diff mbox series

[3/6] net/mlx5: separate Tx function declarations to another file

Message ID 1617631256-3018-4-git-send-email-michaelba@nvidia.com (mailing list archive)
State Superseded
Delegated to: Raslan Darawsheh
Headers show
Series net/mlx5: reduce Tx datapath compile time | expand

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Michael Baum April 5, 2021, 2 p.m. UTC
This patch separates Tx function declarations to different header file
in preparation for removing their implementation from the source file
and as an optional preparation for Tx cleanup.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_mp_os.c |   1 +
 drivers/net/mlx5/linux/mlx5_os.c    |   1 +
 drivers/net/mlx5/linux/mlx5_verbs.c |   2 +-
 drivers/net/mlx5/mlx5.c             |   1 +
 drivers/net/mlx5/mlx5_devx.c        |   2 +-
 drivers/net/mlx5/mlx5_ethdev.c      |   1 +
 drivers/net/mlx5/mlx5_flow.c        |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c     |   2 +-
 drivers/net/mlx5/mlx5_flow_verbs.c  |   1 -
 drivers/net/mlx5/mlx5_mr.c          |   1 +
 drivers/net/mlx5/mlx5_rxmode.c      |   1 -
 drivers/net/mlx5/mlx5_rxq.c         |   2 +-
 drivers/net/mlx5/mlx5_rxtx.c        |   1 +
 drivers/net/mlx5/mlx5_rxtx.h        | 344 ---------------------------------
 drivers/net/mlx5/mlx5_stats.c       |   2 +-
 drivers/net/mlx5/mlx5_trigger.c     |   2 +-
 drivers/net/mlx5/mlx5_tx.h          | 371 ++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_txpp.c        |   2 +-
 drivers/net/mlx5/mlx5_txq.c         |   2 +-
 drivers/net/mlx5/windows/mlx5_os.c  |   1 +
 20 files changed, 387 insertions(+), 355 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_tx.h

Comments

Slava Ovsiienko April 6, 2021, 9:28 a.m. UTC | #1
> -----Original Message-----
> From: Michael Baum <michaelba@nvidia.com>
> Sent: Monday, April 5, 2021 17:01
> To: dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Raslan Darawsheh
> <rasland@nvidia.com>; Slava Ovsiienko <viacheslavo@nvidia.com>
> Subject: [PATCH 3/6] net/mlx5: separate Tx function declarations to another
> file
> 
> This patch separates Tx function declarations to different header file in
> preparation for removing their implementation from the source file and as an
> optional preparation for Tx cleanup.
> 
> Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>

> ---
>  drivers/net/mlx5/linux/mlx5_mp_os.c |   1 +
>  drivers/net/mlx5/linux/mlx5_os.c    |   1 +
>  drivers/net/mlx5/linux/mlx5_verbs.c |   2 +-
>  drivers/net/mlx5/mlx5.c             |   1 +
>  drivers/net/mlx5/mlx5_devx.c        |   2 +-
>  drivers/net/mlx5/mlx5_ethdev.c      |   1 +
>  drivers/net/mlx5/mlx5_flow.c        |   2 +-
>  drivers/net/mlx5/mlx5_flow_dv.c     |   2 +-
>  drivers/net/mlx5/mlx5_flow_verbs.c  |   1 -
>  drivers/net/mlx5/mlx5_mr.c          |   1 +
>  drivers/net/mlx5/mlx5_rxmode.c      |   1 -
>  drivers/net/mlx5/mlx5_rxq.c         |   2 +-
>  drivers/net/mlx5/mlx5_rxtx.c        |   1 +
>  drivers/net/mlx5/mlx5_rxtx.h        | 344 ---------------------------------
>  drivers/net/mlx5/mlx5_stats.c       |   2 +-
>  drivers/net/mlx5/mlx5_trigger.c     |   2 +-
>  drivers/net/mlx5/mlx5_tx.h          | 371
> ++++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/mlx5_txpp.c        |   2 +-
>  drivers/net/mlx5/mlx5_txq.c         |   2 +-
>  drivers/net/mlx5/windows/mlx5_os.c  |   1 +
>  20 files changed, 387 insertions(+), 355 deletions(-)  create mode 100644
> drivers/net/mlx5/mlx5_tx.h
> 
> diff --git a/drivers/net/mlx5/linux/mlx5_mp_os.c
> b/drivers/net/mlx5/linux/mlx5_mp_os.c
> index 63fa278..ca529b6 100644
> --- a/drivers/net/mlx5/linux/mlx5_mp_os.c
> +++ b/drivers/net/mlx5/linux/mlx5_mp_os.c
> @@ -17,6 +17,7 @@
>  #include "mlx5.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_utils.h"
> 
>  int
> diff --git a/drivers/net/mlx5/linux/mlx5_os.c
> b/drivers/net/mlx5/linux/mlx5_os.c
> index 97a28ec..026423b 100644
> --- a/drivers/net/mlx5/linux/mlx5_os.c
> +++ b/drivers/net/mlx5/linux/mlx5_os.c
> @@ -41,6 +41,7 @@
>  #include "mlx5_utils.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
>  #include "mlx5_mr.h"
>  #include "mlx5_flow.h"
> diff --git a/drivers/net/mlx5/linux/mlx5_verbs.c
> b/drivers/net/mlx5/linux/mlx5_verbs.c
> index 73096af..0b0759f 100644
> --- a/drivers/net/mlx5/linux/mlx5_verbs.c
> +++ b/drivers/net/mlx5/linux/mlx5_verbs.c
> @@ -20,9 +20,9 @@
>  #include <mlx5_glue.h>
>  #include <mlx5_common.h>
>  #include <mlx5_common_mr.h>
> -#include <mlx5_rxtx.h>
>  #include <mlx5_verbs.h>
>  #include <mlx5_rx.h>
> +#include <mlx5_tx.h>
>  #include <mlx5_utils.h>
>  #include <mlx5_malloc.h>
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> 6f77bc2..02cc2c7 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -36,6 +36,7 @@
>  #include "mlx5_utils.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
>  #include "mlx5_mr.h"
>  #include "mlx5_flow.h"
> diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
> index 76935f6..76d31f5 100644
> --- a/drivers/net/mlx5/mlx5_devx.c
> +++ b/drivers/net/mlx5/mlx5_devx.c
> @@ -20,7 +20,7 @@
> 
>  #include "mlx5.h"
>  #include "mlx5_common_os.h"
> -#include "mlx5_rxtx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_rx.h"
>  #include "mlx5_utils.h"
>  #include "mlx5_devx.h"
> diff --git a/drivers/net/mlx5/mlx5_ethdev.c
> b/drivers/net/mlx5/mlx5_ethdev.c index 708e3a3..90baee5 100644
> --- a/drivers/net/mlx5/mlx5_ethdev.c
> +++ b/drivers/net/mlx5/mlx5_ethdev.c
> @@ -24,6 +24,7 @@
> 
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
> 
>  /**
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index b3877a1..4dea006 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -29,8 +29,8 @@
>  #include "mlx5.h"
>  #include "mlx5_flow.h"
>  #include "mlx5_flow_os.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_common_os.h"
>  #include "rte_pmd_mlx5.h"
> 
> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c
> b/drivers/net/mlx5/mlx5_flow_dv.c index cac05fb..cb5b3c9 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -32,8 +32,8 @@
>  #include "mlx5_common_os.h"
>  #include "mlx5_flow.h"
>  #include "mlx5_flow_os.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "rte_pmd_mlx5.h"
> 
>  #if defined(HAVE_IBV_FLOW_DV_SUPPORT) ||
> !defined(HAVE_INFINIBAND_VERBS_H) diff --git
> a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index c331350..0fdafbb 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -23,7 +23,6 @@
>  #include "mlx5_defs.h"
>  #include "mlx5.h"
>  #include "mlx5_flow.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> 
>  #define VERBS_SPEC_INNER(item_flags) \
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index
> 2014936..e791b63 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -16,6 +16,7 @@
>  #include "mlx5_mr.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
> 
>  struct mr_find_contig_memsegs_data {
>  	uintptr_t addr;
> diff --git a/drivers/net/mlx5/mlx5_rxmode.c
> b/drivers/net/mlx5/mlx5_rxmode.c index cf93cca..25fb47c 100644
> --- a/drivers/net/mlx5/mlx5_rxmode.c
> +++ b/drivers/net/mlx5/mlx5_rxmode.c
> @@ -11,7 +11,6 @@
> 
>  #include <mlx5_glue.h>
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_utils.h"
> 
>  /**
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index 19df0fa..bb9a908 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -24,7 +24,7 @@
> 
>  #include "mlx5_defs.h"
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_rx.h"
>  #include "mlx5_utils.h"
>  #include "mlx5_autoconf.h"
> diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> index c7f2605..57ff407 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -26,6 +26,7 @@
>  #include "mlx5_utils.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
> 
>  /* TX burst subroutines return codes. */  enum mlx5_txcmp_code { diff --git
> a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index
> f1ebc99..e168dd4 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -17,169 +17,18 @@
>  #include <rte_spinlock.h>
>  #include <rte_io.h>
>  #include <rte_bus_pci.h>
> -#include <rte_malloc.h>
>  #include <rte_cycles.h>
> 
> -#include <mlx5_glue.h>
> -#include <mlx5_prm.h>
>  #include <mlx5_common.h>
>  #include <mlx5_common_mr.h>
> 
> -#include "mlx5_defs.h"
>  #include "mlx5_utils.h"
>  #include "mlx5.h"
>  #include "mlx5_autoconf.h"
>  #include "mlx5_mr.h"
> 
> -
> -/* Mbuf dynamic flag offset for inline. */ -extern uint64_t
> rte_net_mlx5_dynf_inline_mask;
> -
> -struct mlx5_txq_stats {
> -#ifdef MLX5_PMD_SOFT_COUNTERS
> -	uint64_t opackets; /**< Total of successfully sent packets. */
> -	uint64_t obytes; /**< Total of successfully sent bytes. */
> -#endif
> -	uint64_t oerrors; /**< Total number of failed transmitted packets. */
> -};
> -
>  struct mlx5_priv;
> 
> -/* TX queue send local data. */
> -__extension__
> -struct mlx5_txq_local {
> -	struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */
> -	struct rte_mbuf *mbuf; /* first mbuf to process. */
> -	uint16_t pkts_copy; /* packets copied to elts. */
> -	uint16_t pkts_sent; /* packets sent. */
> -	uint16_t pkts_loop; /* packets sent on loop entry. */
> -	uint16_t elts_free; /* available elts remain. */
> -	uint16_t wqe_free; /* available wqe remain. */
> -	uint16_t mbuf_off; /* data offset in current mbuf. */
> -	uint16_t mbuf_nseg; /* number of remaining mbuf. */
> -	uint16_t mbuf_free; /* number of inline mbufs to free. */
> -};
> -
> -/* TX queue descriptor. */
> -__extension__
> -struct mlx5_txq_data {
> -	uint16_t elts_head; /* Current counter in (*elts)[]. */
> -	uint16_t elts_tail; /* Counter of first element awaiting completion. */
> -	uint16_t elts_comp; /* elts index since last completion request. */
> -	uint16_t elts_s; /* Number of mbuf elements. */
> -	uint16_t elts_m; /* Mask for mbuf elements indices. */
> -	/* Fields related to elts mbuf storage. */
> -	uint16_t wqe_ci; /* Consumer index for work queue. */
> -	uint16_t wqe_pi; /* Producer index for work queue. */
> -	uint16_t wqe_s; /* Number of WQ elements. */
> -	uint16_t wqe_m; /* Mask Number for WQ elements. */
> -	uint16_t wqe_comp; /* WQE index since last completion request. */
> -	uint16_t wqe_thres; /* WQE threshold to request completion in CQ.
> */
> -	/* WQ related fields. */
> -	uint16_t cq_ci; /* Consumer index for completion queue. */
> -	uint16_t cq_pi; /* Production index for completion queue. */
> -	uint16_t cqe_s; /* Number of CQ elements. */
> -	uint16_t cqe_m; /* Mask for CQ indices. */
> -	/* CQ related fields. */
> -	uint16_t elts_n:4; /* elts[] length (in log2). */
> -	uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
> -	uint16_t wqe_n:4; /* Number of WQ elements (in log2). */
> -	uint16_t tso_en:1; /* When set hardware TSO is enabled. */
> -	uint16_t tunnel_en:1;
> -	/* When set TX offload for tunneled packets are supported. */
> -	uint16_t swp_en:1; /* Whether SW parser is enabled. */
> -	uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
> -	uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */
> -	uint16_t db_heu:1; /* Doorbell heuristic write barrier. */
> -	uint16_t fast_free:1; /* mbuf fast free on Tx is enabled. */
> -	uint16_t inlen_send; /* Ordinary send data inline size. */
> -	uint16_t inlen_empw; /* eMPW max packet size to inline. */
> -	uint16_t inlen_mode; /* Minimal data length to inline. */
> -	uint32_t qp_num_8s; /* QP number shifted by 8. */
> -	uint64_t offloads; /* Offloads for Tx Queue. */
> -	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
> -	struct mlx5_wqe *wqes; /* Work queue. */
> -	struct mlx5_wqe *wqes_end; /* Work queue array limit. */
> -#ifdef RTE_LIBRTE_MLX5_DEBUG
> -	uint32_t *fcqs; /* Free completion queue (debug extended). */
> -#else
> -	uint16_t *fcqs; /* Free completion queue. */
> -#endif
> -	volatile struct mlx5_cqe *cqes; /* Completion queue. */
> -	volatile uint32_t *qp_db; /* Work queue doorbell. */
> -	volatile uint32_t *cq_db; /* Completion queue doorbell. */
> -	uint16_t port_id; /* Port ID of device. */
> -	uint16_t idx; /* Queue index. */
> -	uint64_t ts_mask; /* Timestamp flag dynamic mask. */
> -	int32_t ts_offset; /* Timestamp field dynamic offset. */
> -	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
> -	struct mlx5_txq_stats stats; /* TX queue counters. */
> -#ifndef RTE_ARCH_64
> -	rte_spinlock_t *uar_lock;
> -	/* UAR access lock required for 32bit implementations */
> -#endif
> -	struct rte_mbuf *elts[0];
> -	/* Storage for queued packets, must be the last field. */
> -} __rte_cache_aligned;
> -
> -enum mlx5_txq_type {
> -	MLX5_TXQ_TYPE_STANDARD, /* Standard Tx queue. */
> -	MLX5_TXQ_TYPE_HAIRPIN, /* Hairpin Rx queue. */
> -};
> -
> -/* TX queue control descriptor. */
> -struct mlx5_txq_ctrl {
> -	LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
> -	uint32_t refcnt; /* Reference counter. */
> -	unsigned int socket; /* CPU socket ID for allocations. */
> -	enum mlx5_txq_type type; /* The txq ctrl type. */
> -	unsigned int max_inline_data; /* Max inline data. */
> -	unsigned int max_tso_header; /* Max TSO header size. */
> -	struct mlx5_txq_obj *obj; /* Verbs/DevX queue object. */
> -	struct mlx5_priv *priv; /* Back pointer to private data. */
> -	off_t uar_mmap_offset; /* UAR mmap offset for non-primary
> process. */
> -	void *bf_reg; /* BlueFlame register from Verbs. */
> -	uint16_t dump_file_n; /* Number of dump files. */
> -	struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
> -	uint32_t hairpin_status; /* Hairpin binding status. */
> -	struct mlx5_txq_data txq; /* Data path structure. */
> -	/* Must be the last field in the structure, contains elts[]. */
> -};
> -
> -#define MLX5_TX_BFREG(txq) \
> -		(MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx])
> -
> -/* mlx5_txq.c */
> -
> -int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id); -int
> mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id); -int
> mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t
> queue_id); -int mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev,
> uint16_t queue_id); -int mlx5_tx_queue_setup(struct rte_eth_dev *dev,
> uint16_t idx, uint16_t desc,
> -			unsigned int socket, const struct rte_eth_txconf
> *conf);
> -int mlx5_tx_hairpin_queue_setup
> -	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
> -	 const struct rte_eth_hairpin_conf *hairpin_conf);
> -void mlx5_tx_queue_release(void *dpdk_txq); -void txq_uar_init(struct
> mlx5_txq_ctrl *txq_ctrl); -int mlx5_tx_uar_init_secondary(struct
> rte_eth_dev *dev, int fd); -void mlx5_tx_uar_uninit_secondary(struct
> rte_eth_dev *dev); -int mlx5_txq_obj_verify(struct rte_eth_dev *dev); -
> struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
> -				   uint16_t desc, unsigned int socket,
> -				   const struct rte_eth_txconf *conf);
> -struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
> -	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
> -	 const struct rte_eth_hairpin_conf *hairpin_conf);
> -struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
> -int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx); -int
> mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx); -int
> mlx5_txq_verify(struct rte_eth_dev *dev); -void txq_alloc_elts(struct
> mlx5_txq_ctrl *txq_ctrl); -void txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl);
> -uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev); -void
> mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
> -
>  /* mlx5_rxtx.c */
> 
>  extern uint32_t mlx5_ptype_table[];
> @@ -189,88 +38,22 @@ struct mlx5_txq_ctrl *mlx5_txq_hairpin_new  void
> mlx5_set_ptype_table(void);  void mlx5_set_cksum_table(void);  void
> mlx5_set_swp_types_table(void); -uint16_t removed_tx_burst(void
> *dpdk_txq, struct rte_mbuf **pkts,
> -			  uint16_t pkts_n);
> -int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);  void
> mlx5_dump_debug_information(const char *path, const char *title,
>  				 const void *buf, unsigned int len);  int
> mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
>  			const struct mlx5_mp_arg_queue_state_modify
> *sm);  int mlx5_queue_state_modify(struct rte_eth_dev *dev,
>  			    struct mlx5_mp_arg_queue_state_modify *sm); -
> void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
> -		       struct rte_eth_txq_info *qinfo);
> -int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t
> tx_queue_id,
> -			   struct rte_eth_burst_mode *mode);
> 
>  /* mlx5_mr.c */
> 
>  void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl); -uint32_t
> mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb); -
> uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t
> addr,
> -			       struct rte_mempool *mp);
>  int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova,
>  		 size_t len);
>  int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t
> iova,
>  		   size_t len);
> 
>  /**
> - * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
> - * 64bit architectures.
> - *
> - * @param val
> - *   value to write in CPU endian format.
> - * @param addr
> - *   Address to write to.
> - * @param lock
> - *   Address of the lock to use for that UAR access.
> - */
> -static __rte_always_inline void
> -__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
> -			   rte_spinlock_t *lock __rte_unused)
> -{
> -#ifdef RTE_ARCH_64
> -	*(uint64_t *)addr = val;
> -#else /* !RTE_ARCH_64 */
> -	rte_spinlock_lock(lock);
> -	*(uint32_t *)addr = val;
> -	rte_io_wmb();
> -	*((uint32_t *)addr + 1) = val >> 32;
> -	rte_spinlock_unlock(lock);
> -#endif
> -}
> -
> -/**
> - * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
> - * 64bit architectures while guaranteeing the order of execution with the
> - * code being executed.
> - *
> - * @param val
> - *   value to write in CPU endian format.
> - * @param addr
> - *   Address to write to.
> - * @param lock
> - *   Address of the lock to use for that UAR access.
> - */
> -static __rte_always_inline void
> -__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock) -{
> -	rte_io_wmb();
> -	__mlx5_uar_write64_relaxed(val, addr, lock);
> -}
> -
> -/* Assist macros, used instead of directly calling the functions they wrap. */
> -#ifdef RTE_ARCH_64 -#define mlx5_uar_write64_relaxed(val, dst, lock) \
> -		__mlx5_uar_write64_relaxed(val, dst, NULL)
> -#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst,
> NULL) -#else -#define mlx5_uar_write64_relaxed(val, dst, lock) \
> -		__mlx5_uar_write64_relaxed(val, dst, lock)
> -#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
> -#endif
> -
> -/**
>   * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from
> which the
>   * cloned mbuf is allocated is returned instead.
>   *
> @@ -288,131 +71,4 @@ int mlx5_dma_unmap(struct rte_pci_device *pdev,
> void *addr, uint64_t iova,
>  	return buf->pool;
>  }
> 
> -/**
> - * Query LKey from a packet buffer for Tx. If not found, add the mempool.
> - *
> - * @param txq
> - *   Pointer to Tx queue structure.
> - * @param addr
> - *   Address to search.
> - *
> - * @return
> - *   Searched LKey on success, UINT32_MAX on no match.
> - */
> -static __rte_always_inline uint32_t
> -mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) -{
> -	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
> -	uintptr_t addr = (uintptr_t)mb->buf_addr;
> -	uint32_t lkey;
> -
> -	/* Check generation bit to see if there's any change on existing MRs.
> */
> -	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
> -		mlx5_mr_flush_local_cache(mr_ctrl);
> -	/* Linear search on MR cache array. */
> -	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
> -				   MLX5_MR_CACHE_N, addr);
> -	if (likely(lkey != UINT32_MAX))
> -		return lkey;
> -	/* Take slower bottom-half on miss. */
> -	return mlx5_tx_mb2mr_bh(txq, mb);
> -}
> -
> -/**
> - * Ring TX queue doorbell and flush the update if requested.
> - *
> - * @param txq
> - *   Pointer to TX queue structure.
> - * @param wqe
> - *   Pointer to the last WQE posted in the NIC.
> - * @param cond
> - *   Request for write memory barrier after BlueFlame update.
> - */
> -static __rte_always_inline void
> -mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct
> mlx5_wqe *wqe,
> -		       int cond)
> -{
> -	uint64_t *dst = MLX5_TX_BFREG(txq);
> -	volatile uint64_t *src = ((volatile uint64_t *)wqe);
> -
> -	rte_io_wmb();
> -	*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
> -	/* Ensure ordering between DB record and BF copy. */
> -	rte_wmb();
> -	mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
> -	if (cond)
> -		rte_wmb();
> -}
> -
> -/**
> - * Ring TX queue doorbell and flush the update by write memory barrier.
> - *
> - * @param txq
> - *   Pointer to TX queue structure.
> - * @param wqe
> - *   Pointer to the last WQE posted in the NIC.
> - */
> -static __rte_always_inline void
> -mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe) -
> {
> -	mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
> -}
> -
> -/**
> - * Convert timestamp from mbuf format to linear counter
> - * of Clock Queue completions (24 bits)
> - *
> - * @param sh
> - *   Pointer to the device shared context to fetch Tx
> - *   packet pacing timestamp and parameters.
> - * @param ts
> - *   Timestamp from mbuf to convert.
> - * @return
> - *   positive or zero value - completion ID to wait
> - *   negative value - conversion error
> - */
> -static __rte_always_inline int32_t
> -mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts) -{
> -	uint64_t ts, ci;
> -	uint32_t tick;
> -
> -	do {
> -		/*
> -		 * Read atomically two uint64_t fields and compare lsb bits.
> -		 * It there is no match - the timestamp was updated in
> -		 * the service thread, data should be re-read.
> -		 */
> -		rte_compiler_barrier();
> -		ci = __atomic_load_n(&sh->txpp.ts.ci_ts,
> __ATOMIC_RELAXED);
> -		ts = __atomic_load_n(&sh->txpp.ts.ts,
> __ATOMIC_RELAXED);
> -		rte_compiler_barrier();
> -		if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
> -			break;
> -	} while (true);
> -	/* Perform the skew correction, positive value to send earlier. */
> -	mts -= sh->txpp.skew;
> -	mts -= ts;
> -	if (unlikely(mts >= UINT64_MAX / 2)) {
> -		/* We have negative integer, mts is in the past. */
> -		__atomic_fetch_add(&sh->txpp.err_ts_past,
> -				   1, __ATOMIC_RELAXED);
> -		return -1;
> -	}
> -	tick = sh->txpp.tick;
> -	MLX5_ASSERT(tick);
> -	/* Convert delta to completions, round up. */
> -	mts = (mts + tick - 1) / tick;
> -	if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
> -		/* We have mts is too distant future. */
> -		__atomic_fetch_add(&sh->txpp.err_ts_future,
> -				   1, __ATOMIC_RELAXED);
> -		return -1;
> -	}
> -	mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
> -	ci += mts;
> -	ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
> -	return ci;
> -}
> -
>  #endif /* RTE_PMD_MLX5_RXTX_H_ */
> diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c
> index 4dbd831..ae2f566 100644
> --- a/drivers/net/mlx5/mlx5_stats.c
> +++ b/drivers/net/mlx5/mlx5_stats.c
> @@ -16,8 +16,8 @@
> 
>  #include "mlx5_defs.h"
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_malloc.h"
> 
>  /**
> diff --git a/drivers/net/mlx5/mlx5_trigger.c
> b/drivers/net/mlx5/mlx5_trigger.c index c88cb22..001c0b5 100644
> --- a/drivers/net/mlx5/mlx5_trigger.c
> +++ b/drivers/net/mlx5/mlx5_trigger.c
> @@ -15,8 +15,8 @@
> 
>  #include "mlx5.h"
>  #include "mlx5_mr.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_utils.h"
>  #include "rte_pmd_mlx5.h"
> 
> diff --git a/drivers/net/mlx5/mlx5_tx.h b/drivers/net/mlx5/mlx5_tx.h new
> file mode 100644 index 0000000..7f91d04
> --- /dev/null
> +++ b/drivers/net/mlx5/mlx5_tx.h
> @@ -0,0 +1,371 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 6WIND S.A.
> + * Copyright 2021 Mellanox Technologies, Ltd  */
> +
> +#ifndef RTE_PMD_MLX5_TX_H_
> +#define RTE_PMD_MLX5_TX_H_
> +
> +#include <stdint.h>
> +#include <sys/queue.h>
> +
> +#include <rte_mbuf.h>
> +#include <rte_mempool.h>
> +#include <rte_common.h>
> +#include <rte_spinlock.h>
> +
> +#include <mlx5_common_mr.h>
> +
> +#include "mlx5.h"
> +#include "mlx5_autoconf.h"
> +#include "mlx5_mr.h"
> +
> +/* Mbuf dynamic flag offset for inline. */ extern uint64_t
> +rte_net_mlx5_dynf_inline_mask;
> +
> +struct mlx5_txq_stats {
> +#ifdef MLX5_PMD_SOFT_COUNTERS
> +	uint64_t opackets; /**< Total of successfully sent packets. */
> +	uint64_t obytes; /**< Total of successfully sent bytes. */ #endif
> +	uint64_t oerrors; /**< Total number of failed transmitted packets. */
> +};
> +
> +/* TX queue send local data. */
> +__extension__
> +struct mlx5_txq_local {
> +	struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */
> +	struct rte_mbuf *mbuf; /* first mbuf to process. */
> +	uint16_t pkts_copy; /* packets copied to elts. */
> +	uint16_t pkts_sent; /* packets sent. */
> +	uint16_t pkts_loop; /* packets sent on loop entry. */
> +	uint16_t elts_free; /* available elts remain. */
> +	uint16_t wqe_free; /* available wqe remain. */
> +	uint16_t mbuf_off; /* data offset in current mbuf. */
> +	uint16_t mbuf_nseg; /* number of remaining mbuf. */
> +	uint16_t mbuf_free; /* number of inline mbufs to free. */ };
> +
> +/* TX queue descriptor. */
> +__extension__
> +struct mlx5_txq_data {
> +	uint16_t elts_head; /* Current counter in (*elts)[]. */
> +	uint16_t elts_tail; /* Counter of first element awaiting completion. */
> +	uint16_t elts_comp; /* elts index since last completion request. */
> +	uint16_t elts_s; /* Number of mbuf elements. */
> +	uint16_t elts_m; /* Mask for mbuf elements indices. */
> +	/* Fields related to elts mbuf storage. */
> +	uint16_t wqe_ci; /* Consumer index for work queue. */
> +	uint16_t wqe_pi; /* Producer index for work queue. */
> +	uint16_t wqe_s; /* Number of WQ elements. */
> +	uint16_t wqe_m; /* Mask Number for WQ elements. */
> +	uint16_t wqe_comp; /* WQE index since last completion request. */
> +	uint16_t wqe_thres; /* WQE threshold to request completion in CQ.
> */
> +	/* WQ related fields. */
> +	uint16_t cq_ci; /* Consumer index for completion queue. */
> +	uint16_t cq_pi; /* Production index for completion queue. */
> +	uint16_t cqe_s; /* Number of CQ elements. */
> +	uint16_t cqe_m; /* Mask for CQ indices. */
> +	/* CQ related fields. */
> +	uint16_t elts_n:4; /* elts[] length (in log2). */
> +	uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
> +	uint16_t wqe_n:4; /* Number of WQ elements (in log2). */
> +	uint16_t tso_en:1; /* When set hardware TSO is enabled. */
> +	uint16_t tunnel_en:1;
> +	/* When set TX offload for tunneled packets are supported. */
> +	uint16_t swp_en:1; /* Whether SW parser is enabled. */
> +	uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
> +	uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */
> +	uint16_t db_heu:1; /* Doorbell heuristic write barrier. */
> +	uint16_t fast_free:1; /* mbuf fast free on Tx is enabled. */
> +	uint16_t inlen_send; /* Ordinary send data inline size. */
> +	uint16_t inlen_empw; /* eMPW max packet size to inline. */
> +	uint16_t inlen_mode; /* Minimal data length to inline. */
> +	uint32_t qp_num_8s; /* QP number shifted by 8. */
> +	uint64_t offloads; /* Offloads for Tx Queue. */
> +	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
> +	struct mlx5_wqe *wqes; /* Work queue. */
> +	struct mlx5_wqe *wqes_end; /* Work queue array limit. */ #ifdef
> +RTE_LIBRTE_MLX5_DEBUG
> +	uint32_t *fcqs; /* Free completion queue (debug extended). */
> #else
> +	uint16_t *fcqs; /* Free completion queue. */ #endif
> +	volatile struct mlx5_cqe *cqes; /* Completion queue. */
> +	volatile uint32_t *qp_db; /* Work queue doorbell. */
> +	volatile uint32_t *cq_db; /* Completion queue doorbell. */
> +	uint16_t port_id; /* Port ID of device. */
> +	uint16_t idx; /* Queue index. */
> +	uint64_t ts_mask; /* Timestamp flag dynamic mask. */
> +	int32_t ts_offset; /* Timestamp field dynamic offset. */
> +	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
> +	struct mlx5_txq_stats stats; /* TX queue counters. */ #ifndef
> +RTE_ARCH_64
> +	rte_spinlock_t *uar_lock;
> +	/* UAR access lock required for 32bit implementations */ #endif
> +	struct rte_mbuf *elts[0];
> +	/* Storage for queued packets, must be the last field. */ }
> +__rte_cache_aligned;
> +
> +enum mlx5_txq_type {
> +	MLX5_TXQ_TYPE_STANDARD, /* Standard Tx queue. */
> +	MLX5_TXQ_TYPE_HAIRPIN, /* Hairpin Tx queue. */ };
> +
> +/* TX queue control descriptor. */
> +struct mlx5_txq_ctrl {
> +	LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
> +	uint32_t refcnt; /* Reference counter. */
> +	unsigned int socket; /* CPU socket ID for allocations. */
> +	enum mlx5_txq_type type; /* The txq ctrl type. */
> +	unsigned int max_inline_data; /* Max inline data. */
> +	unsigned int max_tso_header; /* Max TSO header size. */
> +	struct mlx5_txq_obj *obj; /* Verbs/DevX queue object. */
> +	struct mlx5_priv *priv; /* Back pointer to private data. */
> +	off_t uar_mmap_offset; /* UAR mmap offset for non-primary
> process. */
> +	void *bf_reg; /* BlueFlame register from Verbs. */
> +	uint16_t dump_file_n; /* Number of dump files. */
> +	struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
> +	uint32_t hairpin_status; /* Hairpin binding status. */
> +	struct mlx5_txq_data txq; /* Data path structure. */
> +	/* Must be the last field in the structure, contains elts[]. */ };
> +
> +/* mlx5_txq.c */
> +
> +int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
> +int mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id); int
> +mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t
> +queue_id); int mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev,
> +uint16_t queue_id); int mlx5_tx_queue_setup(struct rte_eth_dev *dev,
> uint16_t idx, uint16_t desc,
> +			unsigned int socket, const struct rte_eth_txconf
> *conf); int
> +mlx5_tx_hairpin_queue_setup
> +	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
> +	 const struct rte_eth_hairpin_conf *hairpin_conf); void
> +mlx5_tx_queue_release(void *dpdk_txq); void txq_uar_init(struct
> +mlx5_txq_ctrl *txq_ctrl); int mlx5_tx_uar_init_secondary(struct
> +rte_eth_dev *dev, int fd); void mlx5_tx_uar_uninit_secondary(struct
> +rte_eth_dev *dev); int mlx5_txq_obj_verify(struct rte_eth_dev *dev);
> +struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t
> idx,
> +				   uint16_t desc, unsigned int socket,
> +				   const struct rte_eth_txconf *conf); struct
> mlx5_txq_ctrl
> +*mlx5_txq_hairpin_new
> +	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
> +	 const struct rte_eth_hairpin_conf *hairpin_conf); struct
> +mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx); int
> +mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx); int
> +mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx); int
> +mlx5_txq_verify(struct rte_eth_dev *dev); void txq_alloc_elts(struct
> +mlx5_txq_ctrl *txq_ctrl); void txq_free_elts(struct mlx5_txq_ctrl
> +*txq_ctrl); uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev
> +*dev); void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
> +
> +/* mlx5_tx.c */
> +
> +uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
> +			  uint16_t pkts_n);
> +int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset); void
> +mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
> +		       struct rte_eth_txq_info *qinfo); int
> +mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
> +			   struct rte_eth_burst_mode *mode);
> +
> +/* mlx5_mr.c */
> +
> +uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf
> +*mb); uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq,
> uintptr_t addr,
> +			       struct rte_mempool *mp);
> +
> +static __rte_always_inline uint64_t *
> +mlx5_tx_bfreg(struct mlx5_txq_data *txq) {
> +	return MLX5_PROC_PRIV(txq->port_id)->uar_table[txq->idx];
> +}
> +
> +/**
> + * Provide safe 64bit store operation to mlx5 UAR region for both 32bit
> +and
> + * 64bit architectures.
> + *
> + * @param val
> + *   value to write in CPU endian format.
> + * @param addr
> + *   Address to write to.
> + * @param lock
> + *   Address of the lock to use for that UAR access.
> + */
> +static __rte_always_inline void
> +__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
> +			   rte_spinlock_t *lock __rte_unused) { #ifdef
> RTE_ARCH_64
> +	*(uint64_t *)addr = val;
> +#else /* !RTE_ARCH_64 */
> +	rte_spinlock_lock(lock);
> +	*(uint32_t *)addr = val;
> +	rte_io_wmb();
> +	*((uint32_t *)addr + 1) = val >> 32;
> +	rte_spinlock_unlock(lock);
> +#endif
> +}
> +
> +/**
> + * Provide safe 64bit store operation to mlx5 UAR region for both 32bit
> +and
> + * 64bit architectures while guaranteeing the order of execution with
> +the
> + * code being executed.
> + *
> + * @param val
> + *   value to write in CPU endian format.
> + * @param addr
> + *   Address to write to.
> + * @param lock
> + *   Address of the lock to use for that UAR access.
> + */
> +static __rte_always_inline void
> +__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock) {
> +	rte_io_wmb();
> +	__mlx5_uar_write64_relaxed(val, addr, lock); }
> +
> +/* Assist macros, used instead of directly calling the functions they
> +wrap. */ #ifdef RTE_ARCH_64 #define mlx5_uar_write64_relaxed(val, dst,
> +lock) \
> +		__mlx5_uar_write64_relaxed(val, dst, NULL) #define
> +mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL)
> +#else #define mlx5_uar_write64_relaxed(val, dst, lock) \
> +		__mlx5_uar_write64_relaxed(val, dst, lock) #define
> +mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
> +#endif
> +
> +/**
> + * Query LKey from a packet buffer for Tx. If not found, add the mempool.
> + *
> + * @param txq
> + *   Pointer to Tx queue structure.
> + * @param addr
> + *   Address to search.
> + *
> + * @return
> + *   Searched LKey on success, UINT32_MAX on no match.
> + */
> +static __rte_always_inline uint32_t
> +mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) {
> +	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
> +	uintptr_t addr = (uintptr_t)mb->buf_addr;
> +	uint32_t lkey;
> +
> +	/* Check generation bit to see if there's any change on existing MRs.
> */
> +	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
> +		mlx5_mr_flush_local_cache(mr_ctrl);
> +	/* Linear search on MR cache array. */
> +	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
> +				   MLX5_MR_CACHE_N, addr);
> +	if (likely(lkey != UINT32_MAX))
> +		return lkey;
> +	/* Take slower bottom-half on miss. */
> +	return mlx5_tx_mb2mr_bh(txq, mb);
> +}
> +
> +/**
> + * Ring TX queue doorbell and flush the update if requested.
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + * @param wqe
> + *   Pointer to the last WQE posted in the NIC.
> + * @param cond
> + *   Request for write memory barrier after BlueFlame update.
> + */
> +static __rte_always_inline void
> +mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct
> mlx5_wqe *wqe,
> +		       int cond)
> +{
> +	uint64_t *dst = mlx5_tx_bfreg(txq);
> +	volatile uint64_t *src = ((volatile uint64_t *)wqe);
> +
> +	rte_io_wmb();
> +	*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
> +	/* Ensure ordering between DB record and BF copy. */
> +	rte_wmb();
> +	mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
> +	if (cond)
> +		rte_wmb();
> +}
> +
> +/**
> + * Ring TX queue doorbell and flush the update by write memory barrier.
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + * @param wqe
> + *   Pointer to the last WQE posted in the NIC.
> + */
> +static __rte_always_inline void
> +mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
> +{
> +	mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
> +}
> +
> +/**
> + * Convert timestamp from mbuf format to linear counter
> + * of Clock Queue completions (24 bits).
> + *
> + * @param sh
> + *   Pointer to the device shared context to fetch Tx
> + *   packet pacing timestamp and parameters.
> + * @param ts
> + *   Timestamp from mbuf to convert.
> + * @return
> + *   positive or zero value - completion ID to wait.
> + *   negative value - conversion error.
> + */
> +static __rte_always_inline int32_t
> +mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts) {
> +	uint64_t ts, ci;
> +	uint32_t tick;
> +
> +	do {
> +		/*
> +		 * Read atomically two uint64_t fields and compare lsb bits.
> +		 * It there is no match - the timestamp was updated in
> +		 * the service thread, data should be re-read.
> +		 */
> +		rte_compiler_barrier();
> +		ci = __atomic_load_n(&sh->txpp.ts.ci_ts,
> __ATOMIC_RELAXED);
> +		ts = __atomic_load_n(&sh->txpp.ts.ts,
> __ATOMIC_RELAXED);
> +		rte_compiler_barrier();
> +		if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
> +			break;
> +	} while (true);
> +	/* Perform the skew correction, positive value to send earlier. */
> +	mts -= sh->txpp.skew;
> +	mts -= ts;
> +	if (unlikely(mts >= UINT64_MAX / 2)) {
> +		/* We have negative integer, mts is in the past. */
> +		__atomic_fetch_add(&sh->txpp.err_ts_past,
> +				   1, __ATOMIC_RELAXED);
> +		return -1;
> +	}
> +	tick = sh->txpp.tick;
> +	MLX5_ASSERT(tick);
> +	/* Convert delta to completions, round up. */
> +	mts = (mts + tick - 1) / tick;
> +	if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
> +		/* We have mts is too distant future. */
> +		__atomic_fetch_add(&sh->txpp.err_ts_future,
> +				   1, __ATOMIC_RELAXED);
> +		return -1;
> +	}
> +	mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
> +	ci += mts;
> +	ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
> +	return ci;
> +}
> +
> +#endif /* RTE_PMD_MLX5_TX_H_ */
> diff --git a/drivers/net/mlx5/mlx5_txpp.c b/drivers/net/mlx5/mlx5_txpp.c
> index 89e1c5d..d90399a 100644
> --- a/drivers/net/mlx5/mlx5_txpp.c
> +++ b/drivers/net/mlx5/mlx5_txpp.c
> @@ -16,8 +16,8 @@
>  #include <mlx5_common_devx.h>
> 
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_common_os.h"
> 
>  static_assert(sizeof(struct mlx5_cqe_ts) == sizeof(rte_int128_t), diff --git
> a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index
> cd13eb9..b8a1657 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -23,7 +23,7 @@
>  #include "mlx5_defs.h"
>  #include "mlx5_utils.h"
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
> 
>  /**
> diff --git a/drivers/net/mlx5/windows/mlx5_os.c
> b/drivers/net/mlx5/windows/mlx5_os.c
> index 79eac80..814063b 100644
> --- a/drivers/net/mlx5/windows/mlx5_os.c
> +++ b/drivers/net/mlx5/windows/mlx5_os.c
> @@ -24,6 +24,7 @@
>  #include "mlx5_utils.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
>  #include "mlx5_mr.h"
>  #include "mlx5_flow.h"
> --
> 1.8.3.1
Raslan Darawsheh April 7, 2021, 11:33 a.m. UTC | #2
Hi Michael,

This patch would cause this compilation failure on aarch64 compilation with gcc :
aarch64-linux-gnu-gcc (Linaro GCC 7.1-2017.08) 7.1.1 20170707

[615/2518] Compiling C object drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_txq.c.o
FAILED: drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_txq.c.o
aarch64-linux-gnu-gcc -Idrivers/libtmp_rte_net_mlx5.a.p -Idrivers -I../../root/dpdk/drivers -Idrivers/net/mlx5 -I../../root/dpdk/drivers/net/mlx5 -Idrivers/net/mlx5/linux -I../../root/dpdk/drivers/net/mlx5/linux -Ilib/librte_ethdev -I../../root/dpdk/lib/librte_ethdev -I. -I../../root/dpdk -Iconfig -I../../root/dpdk/config -Ilib/librte_eal/include -I../../root/dpdk/lib/librte_eal/include -Ilib/librte_eal/linux/include -I../../root/dpdk/lib/librte_eal/linux/include -Ilib/librte_eal/arm/include -I../../root/dpdk/lib/librte_eal/arm/include -Ilib/librte_eal/common -I../../root/dpdk/lib/librte_eal/common -Ilib/librte_eal -I../../root/dpdk/lib/librte_eal -Ilib/librte_kvargs -I../../root/dpdk/lib/librte_kvargs -Ilib/librte_metrics -I../../root/dpdk/lib/librte_metrics -Ilib/librte_telemetry -I../../root/dpdk/lib/librte_telemetry -Ilib/librte_net -I../../root/dpdk/lib/librte_net -Ilib/librte_mbuf -I../../root/dpdk/lib/librte_mbuf -Ilib/librte_mempool -I../../root/dpdk/lib/librte_mempool -Ilib/librte_ring -I../../root/dpdk/lib/librte_ring -Ilib/librte_meter -I../../root/dpdk/lib/librte_meter -Idrivers/bus/pci -I../../root/dpdk/drivers/bus/pci -I../../root/dpdk/drivers/bus/pci/linux -Ilib/librte_pci -I../../root/dpdk/lib/librte_pci -Idrivers/bus/vdev -I../../root/dpdk/drivers/bus/vdev -Ilib/librte_hash -I../../root/dpdk/lib/librte_hash -Ilib/librte_rcu -I../../root/dpdk/lib/librte_rcu -Idrivers/common/mlx5 -I../../root/dpdk/drivers/common/mlx5 -Idrivers/common/mlx5/linux -I../../root/dpdk/drivers/common/mlx5/linux -fdiagnostics-color=always -pipe -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g -include rte_config.h -Wextra -Wcast-qual -Wdeprecated -Wformat -Wformat-nonliteral -Wformat-security -Wmissing-declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition -Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwrite-strings -Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=armv8-a+crc -DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -Wno-format-truncation -std=c11 -Wno-strict-prototypes -D_BSD_SOURCE -D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -pedantic -DPEDANTIC -MD -MQ drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_txq.c.o -MF drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_txq.c.o.d -o drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_txq.c.o -c ../../root/dpdk/drivers/net/mlx5/mlx5_txq.c
../../root/dpdk/drivers/net/mlx5/mlx5_txq.c: In function 'txq_set_params':
../../root/dpdk/drivers/net/mlx5/mlx5_txq.c:817:17: error: dereferencing pointer to incomplete type 'struct rte_pci_device'
   (priv->pci_dev->id.device_id ==
                 ^~        

Kindest regards,
Raslan Darawsheh

> -----Original Message-----
> From: Michael Baum <michaelba@nvidia.com>
> Sent: Monday, April 5, 2021 5:01 PM
> To: dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Raslan Darawsheh
> <rasland@nvidia.com>; Slava Ovsiienko <viacheslavo@nvidia.com>
> Subject: [PATCH 3/6] net/mlx5: separate Tx function declarations to another
> file
> 
> This patch separates Tx function declarations to different header file
> in preparation for removing their implementation from the source file
> and as an optional preparation for Tx cleanup.
> 
> Signed-off-by: Michael Baum <michaelba@nvidia.com>
> ---
>  drivers/net/mlx5/linux/mlx5_mp_os.c |   1 +
>  drivers/net/mlx5/linux/mlx5_os.c    |   1 +
>  drivers/net/mlx5/linux/mlx5_verbs.c |   2 +-
>  drivers/net/mlx5/mlx5.c             |   1 +
>  drivers/net/mlx5/mlx5_devx.c        |   2 +-
>  drivers/net/mlx5/mlx5_ethdev.c      |   1 +
>  drivers/net/mlx5/mlx5_flow.c        |   2 +-
>  drivers/net/mlx5/mlx5_flow_dv.c     |   2 +-
>  drivers/net/mlx5/mlx5_flow_verbs.c  |   1 -
>  drivers/net/mlx5/mlx5_mr.c          |   1 +
>  drivers/net/mlx5/mlx5_rxmode.c      |   1 -
>  drivers/net/mlx5/mlx5_rxq.c         |   2 +-
>  drivers/net/mlx5/mlx5_rxtx.c        |   1 +
>  drivers/net/mlx5/mlx5_rxtx.h        | 344 ---------------------------------
>  drivers/net/mlx5/mlx5_stats.c       |   2 +-
>  drivers/net/mlx5/mlx5_trigger.c     |   2 +-
>  drivers/net/mlx5/mlx5_tx.h          | 371
> ++++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/mlx5_txpp.c        |   2 +-
>  drivers/net/mlx5/mlx5_txq.c         |   2 +-
>  drivers/net/mlx5/windows/mlx5_os.c  |   1 +
>  20 files changed, 387 insertions(+), 355 deletions(-)
>  create mode 100644 drivers/net/mlx5/mlx5_tx.h
> 
> diff --git a/drivers/net/mlx5/linux/mlx5_mp_os.c
> b/drivers/net/mlx5/linux/mlx5_mp_os.c
> index 63fa278..ca529b6 100644
> --- a/drivers/net/mlx5/linux/mlx5_mp_os.c
> +++ b/drivers/net/mlx5/linux/mlx5_mp_os.c
> @@ -17,6 +17,7 @@
>  #include "mlx5.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_utils.h"
> 
>  int
> diff --git a/drivers/net/mlx5/linux/mlx5_os.c
> b/drivers/net/mlx5/linux/mlx5_os.c
> index 97a28ec..026423b 100644
> --- a/drivers/net/mlx5/linux/mlx5_os.c
> +++ b/drivers/net/mlx5/linux/mlx5_os.c
> @@ -41,6 +41,7 @@
>  #include "mlx5_utils.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
>  #include "mlx5_mr.h"
>  #include "mlx5_flow.h"
> diff --git a/drivers/net/mlx5/linux/mlx5_verbs.c
> b/drivers/net/mlx5/linux/mlx5_verbs.c
> index 73096af..0b0759f 100644
> --- a/drivers/net/mlx5/linux/mlx5_verbs.c
> +++ b/drivers/net/mlx5/linux/mlx5_verbs.c
> @@ -20,9 +20,9 @@
>  #include <mlx5_glue.h>
>  #include <mlx5_common.h>
>  #include <mlx5_common_mr.h>
> -#include <mlx5_rxtx.h>
>  #include <mlx5_verbs.h>
>  #include <mlx5_rx.h>
> +#include <mlx5_tx.h>
>  #include <mlx5_utils.h>
>  #include <mlx5_malloc.h>
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index 6f77bc2..02cc2c7 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -36,6 +36,7 @@
>  #include "mlx5_utils.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
>  #include "mlx5_mr.h"
>  #include "mlx5_flow.h"
> diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
> index 76935f6..76d31f5 100644
> --- a/drivers/net/mlx5/mlx5_devx.c
> +++ b/drivers/net/mlx5/mlx5_devx.c
> @@ -20,7 +20,7 @@
> 
>  #include "mlx5.h"
>  #include "mlx5_common_os.h"
> -#include "mlx5_rxtx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_rx.h"
>  #include "mlx5_utils.h"
>  #include "mlx5_devx.h"
> diff --git a/drivers/net/mlx5/mlx5_ethdev.c
> b/drivers/net/mlx5/mlx5_ethdev.c
> index 708e3a3..90baee5 100644
> --- a/drivers/net/mlx5/mlx5_ethdev.c
> +++ b/drivers/net/mlx5/mlx5_ethdev.c
> @@ -24,6 +24,7 @@
> 
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
> 
>  /**
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index b3877a1..4dea006 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -29,8 +29,8 @@
>  #include "mlx5.h"
>  #include "mlx5_flow.h"
>  #include "mlx5_flow_os.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_common_os.h"
>  #include "rte_pmd_mlx5.h"
> 
> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c
> b/drivers/net/mlx5/mlx5_flow_dv.c
> index cac05fb..cb5b3c9 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -32,8 +32,8 @@
>  #include "mlx5_common_os.h"
>  #include "mlx5_flow.h"
>  #include "mlx5_flow_os.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "rte_pmd_mlx5.h"
> 
>  #if defined(HAVE_IBV_FLOW_DV_SUPPORT) ||
> !defined(HAVE_INFINIBAND_VERBS_H)
> diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index c331350..0fdafbb 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -23,7 +23,6 @@
>  #include "mlx5_defs.h"
>  #include "mlx5.h"
>  #include "mlx5_flow.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> 
>  #define VERBS_SPEC_INNER(item_flags) \
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> index 2014936..e791b63 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -16,6 +16,7 @@
>  #include "mlx5_mr.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
> 
>  struct mr_find_contig_memsegs_data {
>  	uintptr_t addr;
> diff --git a/drivers/net/mlx5/mlx5_rxmode.c
> b/drivers/net/mlx5/mlx5_rxmode.c
> index cf93cca..25fb47c 100644
> --- a/drivers/net/mlx5/mlx5_rxmode.c
> +++ b/drivers/net/mlx5/mlx5_rxmode.c
> @@ -11,7 +11,6 @@
> 
>  #include <mlx5_glue.h>
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_utils.h"
> 
>  /**
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index 19df0fa..bb9a908 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -24,7 +24,7 @@
> 
>  #include "mlx5_defs.h"
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_rx.h"
>  #include "mlx5_utils.h"
>  #include "mlx5_autoconf.h"
> diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> index c7f2605..57ff407 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -26,6 +26,7 @@
>  #include "mlx5_utils.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
> 
>  /* TX burst subroutines return codes. */
>  enum mlx5_txcmp_code {
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index f1ebc99..e168dd4 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -17,169 +17,18 @@
>  #include <rte_spinlock.h>
>  #include <rte_io.h>
>  #include <rte_bus_pci.h>
> -#include <rte_malloc.h>
>  #include <rte_cycles.h>
> 
> -#include <mlx5_glue.h>
> -#include <mlx5_prm.h>
>  #include <mlx5_common.h>
>  #include <mlx5_common_mr.h>
> 
> -#include "mlx5_defs.h"
>  #include "mlx5_utils.h"
>  #include "mlx5.h"
>  #include "mlx5_autoconf.h"
>  #include "mlx5_mr.h"
> 
> -
> -/* Mbuf dynamic flag offset for inline. */
> -extern uint64_t rte_net_mlx5_dynf_inline_mask;
> -
> -struct mlx5_txq_stats {
> -#ifdef MLX5_PMD_SOFT_COUNTERS
> -	uint64_t opackets; /**< Total of successfully sent packets. */
> -	uint64_t obytes; /**< Total of successfully sent bytes. */
> -#endif
> -	uint64_t oerrors; /**< Total number of failed transmitted packets. */
> -};
> -
>  struct mlx5_priv;
> 
> -/* TX queue send local data. */
> -__extension__
> -struct mlx5_txq_local {
> -	struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */
> -	struct rte_mbuf *mbuf; /* first mbuf to process. */
> -	uint16_t pkts_copy; /* packets copied to elts. */
> -	uint16_t pkts_sent; /* packets sent. */
> -	uint16_t pkts_loop; /* packets sent on loop entry. */
> -	uint16_t elts_free; /* available elts remain. */
> -	uint16_t wqe_free; /* available wqe remain. */
> -	uint16_t mbuf_off; /* data offset in current mbuf. */
> -	uint16_t mbuf_nseg; /* number of remaining mbuf. */
> -	uint16_t mbuf_free; /* number of inline mbufs to free. */
> -};
> -
> -/* TX queue descriptor. */
> -__extension__
> -struct mlx5_txq_data {
> -	uint16_t elts_head; /* Current counter in (*elts)[]. */
> -	uint16_t elts_tail; /* Counter of first element awaiting completion. */
> -	uint16_t elts_comp; /* elts index since last completion request. */
> -	uint16_t elts_s; /* Number of mbuf elements. */
> -	uint16_t elts_m; /* Mask for mbuf elements indices. */
> -	/* Fields related to elts mbuf storage. */
> -	uint16_t wqe_ci; /* Consumer index for work queue. */
> -	uint16_t wqe_pi; /* Producer index for work queue. */
> -	uint16_t wqe_s; /* Number of WQ elements. */
> -	uint16_t wqe_m; /* Mask Number for WQ elements. */
> -	uint16_t wqe_comp; /* WQE index since last completion request. */
> -	uint16_t wqe_thres; /* WQE threshold to request completion in CQ.
> */
> -	/* WQ related fields. */
> -	uint16_t cq_ci; /* Consumer index for completion queue. */
> -	uint16_t cq_pi; /* Production index for completion queue. */
> -	uint16_t cqe_s; /* Number of CQ elements. */
> -	uint16_t cqe_m; /* Mask for CQ indices. */
> -	/* CQ related fields. */
> -	uint16_t elts_n:4; /* elts[] length (in log2). */
> -	uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
> -	uint16_t wqe_n:4; /* Number of WQ elements (in log2). */
> -	uint16_t tso_en:1; /* When set hardware TSO is enabled. */
> -	uint16_t tunnel_en:1;
> -	/* When set TX offload for tunneled packets are supported. */
> -	uint16_t swp_en:1; /* Whether SW parser is enabled. */
> -	uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
> -	uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */
> -	uint16_t db_heu:1; /* Doorbell heuristic write barrier. */
> -	uint16_t fast_free:1; /* mbuf fast free on Tx is enabled. */
> -	uint16_t inlen_send; /* Ordinary send data inline size. */
> -	uint16_t inlen_empw; /* eMPW max packet size to inline. */
> -	uint16_t inlen_mode; /* Minimal data length to inline. */
> -	uint32_t qp_num_8s; /* QP number shifted by 8. */
> -	uint64_t offloads; /* Offloads for Tx Queue. */
> -	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
> -	struct mlx5_wqe *wqes; /* Work queue. */
> -	struct mlx5_wqe *wqes_end; /* Work queue array limit. */
> -#ifdef RTE_LIBRTE_MLX5_DEBUG
> -	uint32_t *fcqs; /* Free completion queue (debug extended). */
> -#else
> -	uint16_t *fcqs; /* Free completion queue. */
> -#endif
> -	volatile struct mlx5_cqe *cqes; /* Completion queue. */
> -	volatile uint32_t *qp_db; /* Work queue doorbell. */
> -	volatile uint32_t *cq_db; /* Completion queue doorbell. */
> -	uint16_t port_id; /* Port ID of device. */
> -	uint16_t idx; /* Queue index. */
> -	uint64_t ts_mask; /* Timestamp flag dynamic mask. */
> -	int32_t ts_offset; /* Timestamp field dynamic offset. */
> -	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
> -	struct mlx5_txq_stats stats; /* TX queue counters. */
> -#ifndef RTE_ARCH_64
> -	rte_spinlock_t *uar_lock;
> -	/* UAR access lock required for 32bit implementations */
> -#endif
> -	struct rte_mbuf *elts[0];
> -	/* Storage for queued packets, must be the last field. */
> -} __rte_cache_aligned;
> -
> -enum mlx5_txq_type {
> -	MLX5_TXQ_TYPE_STANDARD, /* Standard Tx queue. */
> -	MLX5_TXQ_TYPE_HAIRPIN, /* Hairpin Rx queue. */
> -};
> -
> -/* TX queue control descriptor. */
> -struct mlx5_txq_ctrl {
> -	LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
> -	uint32_t refcnt; /* Reference counter. */
> -	unsigned int socket; /* CPU socket ID for allocations. */
> -	enum mlx5_txq_type type; /* The txq ctrl type. */
> -	unsigned int max_inline_data; /* Max inline data. */
> -	unsigned int max_tso_header; /* Max TSO header size. */
> -	struct mlx5_txq_obj *obj; /* Verbs/DevX queue object. */
> -	struct mlx5_priv *priv; /* Back pointer to private data. */
> -	off_t uar_mmap_offset; /* UAR mmap offset for non-primary
> process. */
> -	void *bf_reg; /* BlueFlame register from Verbs. */
> -	uint16_t dump_file_n; /* Number of dump files. */
> -	struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
> -	uint32_t hairpin_status; /* Hairpin binding status. */
> -	struct mlx5_txq_data txq; /* Data path structure. */
> -	/* Must be the last field in the structure, contains elts[]. */
> -};
> -
> -#define MLX5_TX_BFREG(txq) \
> -		(MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx])
> -
> -/* mlx5_txq.c */
> -
> -int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
> -int mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id);
> -int mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t
> queue_id);
> -int mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t
> queue_id);
> -int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t
> desc,
> -			unsigned int socket, const struct rte_eth_txconf
> *conf);
> -int mlx5_tx_hairpin_queue_setup
> -	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
> -	 const struct rte_eth_hairpin_conf *hairpin_conf);
> -void mlx5_tx_queue_release(void *dpdk_txq);
> -void txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl);
> -int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd);
> -void mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev);
> -int mlx5_txq_obj_verify(struct rte_eth_dev *dev);
> -struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
> -				   uint16_t desc, unsigned int socket,
> -				   const struct rte_eth_txconf *conf);
> -struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
> -	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
> -	 const struct rte_eth_hairpin_conf *hairpin_conf);
> -struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
> -int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
> -int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
> -int mlx5_txq_verify(struct rte_eth_dev *dev);
> -void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
> -void txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl);
> -uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
> -void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
> -
>  /* mlx5_rxtx.c */
> 
>  extern uint32_t mlx5_ptype_table[];
> @@ -189,88 +38,22 @@ struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
>  void mlx5_set_ptype_table(void);
>  void mlx5_set_cksum_table(void);
>  void mlx5_set_swp_types_table(void);
> -uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
> -			  uint16_t pkts_n);
> -int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
>  void mlx5_dump_debug_information(const char *path, const char *title,
>  				 const void *buf, unsigned int len);
>  int mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
>  			const struct mlx5_mp_arg_queue_state_modify
> *sm);
>  int mlx5_queue_state_modify(struct rte_eth_dev *dev,
>  			    struct mlx5_mp_arg_queue_state_modify *sm);
> -void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
> -		       struct rte_eth_txq_info *qinfo);
> -int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t
> tx_queue_id,
> -			   struct rte_eth_burst_mode *mode);
> 
>  /* mlx5_mr.c */
> 
>  void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
> -uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf
> *mb);
> -uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t
> addr,
> -			       struct rte_mempool *mp);
>  int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova,
>  		 size_t len);
>  int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t
> iova,
>  		   size_t len);
> 
>  /**
> - * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
> - * 64bit architectures.
> - *
> - * @param val
> - *   value to write in CPU endian format.
> - * @param addr
> - *   Address to write to.
> - * @param lock
> - *   Address of the lock to use for that UAR access.
> - */
> -static __rte_always_inline void
> -__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
> -			   rte_spinlock_t *lock __rte_unused)
> -{
> -#ifdef RTE_ARCH_64
> -	*(uint64_t *)addr = val;
> -#else /* !RTE_ARCH_64 */
> -	rte_spinlock_lock(lock);
> -	*(uint32_t *)addr = val;
> -	rte_io_wmb();
> -	*((uint32_t *)addr + 1) = val >> 32;
> -	rte_spinlock_unlock(lock);
> -#endif
> -}
> -
> -/**
> - * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
> - * 64bit architectures while guaranteeing the order of execution with the
> - * code being executed.
> - *
> - * @param val
> - *   value to write in CPU endian format.
> - * @param addr
> - *   Address to write to.
> - * @param lock
> - *   Address of the lock to use for that UAR access.
> - */
> -static __rte_always_inline void
> -__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
> -{
> -	rte_io_wmb();
> -	__mlx5_uar_write64_relaxed(val, addr, lock);
> -}
> -
> -/* Assist macros, used instead of directly calling the functions they wrap. */
> -#ifdef RTE_ARCH_64
> -#define mlx5_uar_write64_relaxed(val, dst, lock) \
> -		__mlx5_uar_write64_relaxed(val, dst, NULL)
> -#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst,
> NULL)
> -#else
> -#define mlx5_uar_write64_relaxed(val, dst, lock) \
> -		__mlx5_uar_write64_relaxed(val, dst, lock)
> -#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
> -#endif
> -
> -/**
>   * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from
> which the
>   * cloned mbuf is allocated is returned instead.
>   *
> @@ -288,131 +71,4 @@ int mlx5_dma_unmap(struct rte_pci_device *pdev,
> void *addr, uint64_t iova,
>  	return buf->pool;
>  }
> 
> -/**
> - * Query LKey from a packet buffer for Tx. If not found, add the mempool.
> - *
> - * @param txq
> - *   Pointer to Tx queue structure.
> - * @param addr
> - *   Address to search.
> - *
> - * @return
> - *   Searched LKey on success, UINT32_MAX on no match.
> - */
> -static __rte_always_inline uint32_t
> -mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
> -{
> -	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
> -	uintptr_t addr = (uintptr_t)mb->buf_addr;
> -	uint32_t lkey;
> -
> -	/* Check generation bit to see if there's any change on existing MRs.
> */
> -	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
> -		mlx5_mr_flush_local_cache(mr_ctrl);
> -	/* Linear search on MR cache array. */
> -	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
> -				   MLX5_MR_CACHE_N, addr);
> -	if (likely(lkey != UINT32_MAX))
> -		return lkey;
> -	/* Take slower bottom-half on miss. */
> -	return mlx5_tx_mb2mr_bh(txq, mb);
> -}
> -
> -/**
> - * Ring TX queue doorbell and flush the update if requested.
> - *
> - * @param txq
> - *   Pointer to TX queue structure.
> - * @param wqe
> - *   Pointer to the last WQE posted in the NIC.
> - * @param cond
> - *   Request for write memory barrier after BlueFlame update.
> - */
> -static __rte_always_inline void
> -mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct
> mlx5_wqe *wqe,
> -		       int cond)
> -{
> -	uint64_t *dst = MLX5_TX_BFREG(txq);
> -	volatile uint64_t *src = ((volatile uint64_t *)wqe);
> -
> -	rte_io_wmb();
> -	*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
> -	/* Ensure ordering between DB record and BF copy. */
> -	rte_wmb();
> -	mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
> -	if (cond)
> -		rte_wmb();
> -}
> -
> -/**
> - * Ring TX queue doorbell and flush the update by write memory barrier.
> - *
> - * @param txq
> - *   Pointer to TX queue structure.
> - * @param wqe
> - *   Pointer to the last WQE posted in the NIC.
> - */
> -static __rte_always_inline void
> -mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
> -{
> -	mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
> -}
> -
> -/**
> - * Convert timestamp from mbuf format to linear counter
> - * of Clock Queue completions (24 bits)
> - *
> - * @param sh
> - *   Pointer to the device shared context to fetch Tx
> - *   packet pacing timestamp and parameters.
> - * @param ts
> - *   Timestamp from mbuf to convert.
> - * @return
> - *   positive or zero value - completion ID to wait
> - *   negative value - conversion error
> - */
> -static __rte_always_inline int32_t
> -mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts)
> -{
> -	uint64_t ts, ci;
> -	uint32_t tick;
> -
> -	do {
> -		/*
> -		 * Read atomically two uint64_t fields and compare lsb bits.
> -		 * It there is no match - the timestamp was updated in
> -		 * the service thread, data should be re-read.
> -		 */
> -		rte_compiler_barrier();
> -		ci = __atomic_load_n(&sh->txpp.ts.ci_ts,
> __ATOMIC_RELAXED);
> -		ts = __atomic_load_n(&sh->txpp.ts.ts,
> __ATOMIC_RELAXED);
> -		rte_compiler_barrier();
> -		if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
> -			break;
> -	} while (true);
> -	/* Perform the skew correction, positive value to send earlier. */
> -	mts -= sh->txpp.skew;
> -	mts -= ts;
> -	if (unlikely(mts >= UINT64_MAX / 2)) {
> -		/* We have negative integer, mts is in the past. */
> -		__atomic_fetch_add(&sh->txpp.err_ts_past,
> -				   1, __ATOMIC_RELAXED);
> -		return -1;
> -	}
> -	tick = sh->txpp.tick;
> -	MLX5_ASSERT(tick);
> -	/* Convert delta to completions, round up. */
> -	mts = (mts + tick - 1) / tick;
> -	if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
> -		/* We have mts is too distant future. */
> -		__atomic_fetch_add(&sh->txpp.err_ts_future,
> -				   1, __ATOMIC_RELAXED);
> -		return -1;
> -	}
> -	mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
> -	ci += mts;
> -	ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
> -	return ci;
> -}
> -
>  #endif /* RTE_PMD_MLX5_RXTX_H_ */
> diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c
> index 4dbd831..ae2f566 100644
> --- a/drivers/net/mlx5/mlx5_stats.c
> +++ b/drivers/net/mlx5/mlx5_stats.c
> @@ -16,8 +16,8 @@
> 
>  #include "mlx5_defs.h"
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_malloc.h"
> 
>  /**
> diff --git a/drivers/net/mlx5/mlx5_trigger.c
> b/drivers/net/mlx5/mlx5_trigger.c
> index c88cb22..001c0b5 100644
> --- a/drivers/net/mlx5/mlx5_trigger.c
> +++ b/drivers/net/mlx5/mlx5_trigger.c
> @@ -15,8 +15,8 @@
> 
>  #include "mlx5.h"
>  #include "mlx5_mr.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_utils.h"
>  #include "rte_pmd_mlx5.h"
> 
> diff --git a/drivers/net/mlx5/mlx5_tx.h b/drivers/net/mlx5/mlx5_tx.h
> new file mode 100644
> index 0000000..7f91d04
> --- /dev/null
> +++ b/drivers/net/mlx5/mlx5_tx.h
> @@ -0,0 +1,371 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 6WIND S.A.
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef RTE_PMD_MLX5_TX_H_
> +#define RTE_PMD_MLX5_TX_H_
> +
> +#include <stdint.h>
> +#include <sys/queue.h>
> +
> +#include <rte_mbuf.h>
> +#include <rte_mempool.h>
> +#include <rte_common.h>
> +#include <rte_spinlock.h>
> +
> +#include <mlx5_common_mr.h>
> +
> +#include "mlx5.h"
> +#include "mlx5_autoconf.h"
> +#include "mlx5_mr.h"
> +
> +/* Mbuf dynamic flag offset for inline. */
> +extern uint64_t rte_net_mlx5_dynf_inline_mask;
> +
> +struct mlx5_txq_stats {
> +#ifdef MLX5_PMD_SOFT_COUNTERS
> +	uint64_t opackets; /**< Total of successfully sent packets. */
> +	uint64_t obytes; /**< Total of successfully sent bytes. */
> +#endif
> +	uint64_t oerrors; /**< Total number of failed transmitted packets. */
> +};
> +
> +/* TX queue send local data. */
> +__extension__
> +struct mlx5_txq_local {
> +	struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */
> +	struct rte_mbuf *mbuf; /* first mbuf to process. */
> +	uint16_t pkts_copy; /* packets copied to elts. */
> +	uint16_t pkts_sent; /* packets sent. */
> +	uint16_t pkts_loop; /* packets sent on loop entry. */
> +	uint16_t elts_free; /* available elts remain. */
> +	uint16_t wqe_free; /* available wqe remain. */
> +	uint16_t mbuf_off; /* data offset in current mbuf. */
> +	uint16_t mbuf_nseg; /* number of remaining mbuf. */
> +	uint16_t mbuf_free; /* number of inline mbufs to free. */
> +};
> +
> +/* TX queue descriptor. */
> +__extension__
> +struct mlx5_txq_data {
> +	uint16_t elts_head; /* Current counter in (*elts)[]. */
> +	uint16_t elts_tail; /* Counter of first element awaiting completion. */
> +	uint16_t elts_comp; /* elts index since last completion request. */
> +	uint16_t elts_s; /* Number of mbuf elements. */
> +	uint16_t elts_m; /* Mask for mbuf elements indices. */
> +	/* Fields related to elts mbuf storage. */
> +	uint16_t wqe_ci; /* Consumer index for work queue. */
> +	uint16_t wqe_pi; /* Producer index for work queue. */
> +	uint16_t wqe_s; /* Number of WQ elements. */
> +	uint16_t wqe_m; /* Mask Number for WQ elements. */
> +	uint16_t wqe_comp; /* WQE index since last completion request. */
> +	uint16_t wqe_thres; /* WQE threshold to request completion in CQ.
> */
> +	/* WQ related fields. */
> +	uint16_t cq_ci; /* Consumer index for completion queue. */
> +	uint16_t cq_pi; /* Production index for completion queue. */
> +	uint16_t cqe_s; /* Number of CQ elements. */
> +	uint16_t cqe_m; /* Mask for CQ indices. */
> +	/* CQ related fields. */
> +	uint16_t elts_n:4; /* elts[] length (in log2). */
> +	uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
> +	uint16_t wqe_n:4; /* Number of WQ elements (in log2). */
> +	uint16_t tso_en:1; /* When set hardware TSO is enabled. */
> +	uint16_t tunnel_en:1;
> +	/* When set TX offload for tunneled packets are supported. */
> +	uint16_t swp_en:1; /* Whether SW parser is enabled. */
> +	uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
> +	uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */
> +	uint16_t db_heu:1; /* Doorbell heuristic write barrier. */
> +	uint16_t fast_free:1; /* mbuf fast free on Tx is enabled. */
> +	uint16_t inlen_send; /* Ordinary send data inline size. */
> +	uint16_t inlen_empw; /* eMPW max packet size to inline. */
> +	uint16_t inlen_mode; /* Minimal data length to inline. */
> +	uint32_t qp_num_8s; /* QP number shifted by 8. */
> +	uint64_t offloads; /* Offloads for Tx Queue. */
> +	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
> +	struct mlx5_wqe *wqes; /* Work queue. */
> +	struct mlx5_wqe *wqes_end; /* Work queue array limit. */
> +#ifdef RTE_LIBRTE_MLX5_DEBUG
> +	uint32_t *fcqs; /* Free completion queue (debug extended). */
> +#else
> +	uint16_t *fcqs; /* Free completion queue. */
> +#endif
> +	volatile struct mlx5_cqe *cqes; /* Completion queue. */
> +	volatile uint32_t *qp_db; /* Work queue doorbell. */
> +	volatile uint32_t *cq_db; /* Completion queue doorbell. */
> +	uint16_t port_id; /* Port ID of device. */
> +	uint16_t idx; /* Queue index. */
> +	uint64_t ts_mask; /* Timestamp flag dynamic mask. */
> +	int32_t ts_offset; /* Timestamp field dynamic offset. */
> +	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
> +	struct mlx5_txq_stats stats; /* TX queue counters. */
> +#ifndef RTE_ARCH_64
> +	rte_spinlock_t *uar_lock;
> +	/* UAR access lock required for 32bit implementations */
> +#endif
> +	struct rte_mbuf *elts[0];
> +	/* Storage for queued packets, must be the last field. */
> +} __rte_cache_aligned;
> +
> +enum mlx5_txq_type {
> +	MLX5_TXQ_TYPE_STANDARD, /* Standard Tx queue. */
> +	MLX5_TXQ_TYPE_HAIRPIN, /* Hairpin Tx queue. */
> +};
> +
> +/* TX queue control descriptor. */
> +struct mlx5_txq_ctrl {
> +	LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
> +	uint32_t refcnt; /* Reference counter. */
> +	unsigned int socket; /* CPU socket ID for allocations. */
> +	enum mlx5_txq_type type; /* The txq ctrl type. */
> +	unsigned int max_inline_data; /* Max inline data. */
> +	unsigned int max_tso_header; /* Max TSO header size. */
> +	struct mlx5_txq_obj *obj; /* Verbs/DevX queue object. */
> +	struct mlx5_priv *priv; /* Back pointer to private data. */
> +	off_t uar_mmap_offset; /* UAR mmap offset for non-primary
> process. */
> +	void *bf_reg; /* BlueFlame register from Verbs. */
> +	uint16_t dump_file_n; /* Number of dump files. */
> +	struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
> +	uint32_t hairpin_status; /* Hairpin binding status. */
> +	struct mlx5_txq_data txq; /* Data path structure. */
> +	/* Must be the last field in the structure, contains elts[]. */
> +};
> +
> +/* mlx5_txq.c */
> +
> +int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
> +int mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id);
> +int mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t
> queue_id);
> +int mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t
> queue_id);
> +int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t
> desc,
> +			unsigned int socket, const struct rte_eth_txconf
> *conf);
> +int mlx5_tx_hairpin_queue_setup
> +	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
> +	 const struct rte_eth_hairpin_conf *hairpin_conf);
> +void mlx5_tx_queue_release(void *dpdk_txq);
> +void txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl);
> +int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd);
> +void mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev);
> +int mlx5_txq_obj_verify(struct rte_eth_dev *dev);
> +struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t
> idx,
> +				   uint16_t desc, unsigned int socket,
> +				   const struct rte_eth_txconf *conf);
> +struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
> +	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
> +	 const struct rte_eth_hairpin_conf *hairpin_conf);
> +struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
> +int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
> +int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
> +int mlx5_txq_verify(struct rte_eth_dev *dev);
> +void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
> +void txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl);
> +uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
> +void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
> +
> +/* mlx5_tx.c */
> +
> +uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
> +			  uint16_t pkts_n);
> +int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
> +void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
> +		       struct rte_eth_txq_info *qinfo);
> +int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t
> tx_queue_id,
> +			   struct rte_eth_burst_mode *mode);
> +
> +/* mlx5_mr.c */
> +
> +uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf
> *mb);
> +uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t
> addr,
> +			       struct rte_mempool *mp);
> +
> +static __rte_always_inline uint64_t *
> +mlx5_tx_bfreg(struct mlx5_txq_data *txq)
> +{
> +	return MLX5_PROC_PRIV(txq->port_id)->uar_table[txq->idx];
> +}
> +
> +/**
> + * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
> + * 64bit architectures.
> + *
> + * @param val
> + *   value to write in CPU endian format.
> + * @param addr
> + *   Address to write to.
> + * @param lock
> + *   Address of the lock to use for that UAR access.
> + */
> +static __rte_always_inline void
> +__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
> +			   rte_spinlock_t *lock __rte_unused)
> +{
> +#ifdef RTE_ARCH_64
> +	*(uint64_t *)addr = val;
> +#else /* !RTE_ARCH_64 */
> +	rte_spinlock_lock(lock);
> +	*(uint32_t *)addr = val;
> +	rte_io_wmb();
> +	*((uint32_t *)addr + 1) = val >> 32;
> +	rte_spinlock_unlock(lock);
> +#endif
> +}
> +
> +/**
> + * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
> + * 64bit architectures while guaranteeing the order of execution with the
> + * code being executed.
> + *
> + * @param val
> + *   value to write in CPU endian format.
> + * @param addr
> + *   Address to write to.
> + * @param lock
> + *   Address of the lock to use for that UAR access.
> + */
> +static __rte_always_inline void
> +__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
> +{
> +	rte_io_wmb();
> +	__mlx5_uar_write64_relaxed(val, addr, lock);
> +}
> +
> +/* Assist macros, used instead of directly calling the functions they wrap. */
> +#ifdef RTE_ARCH_64
> +#define mlx5_uar_write64_relaxed(val, dst, lock) \
> +		__mlx5_uar_write64_relaxed(val, dst, NULL)
> +#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst,
> NULL)
> +#else
> +#define mlx5_uar_write64_relaxed(val, dst, lock) \
> +		__mlx5_uar_write64_relaxed(val, dst, lock)
> +#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
> +#endif
> +
> +/**
> + * Query LKey from a packet buffer for Tx. If not found, add the mempool.
> + *
> + * @param txq
> + *   Pointer to Tx queue structure.
> + * @param addr
> + *   Address to search.
> + *
> + * @return
> + *   Searched LKey on success, UINT32_MAX on no match.
> + */
> +static __rte_always_inline uint32_t
> +mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
> +{
> +	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
> +	uintptr_t addr = (uintptr_t)mb->buf_addr;
> +	uint32_t lkey;
> +
> +	/* Check generation bit to see if there's any change on existing MRs.
> */
> +	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
> +		mlx5_mr_flush_local_cache(mr_ctrl);
> +	/* Linear search on MR cache array. */
> +	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
> +				   MLX5_MR_CACHE_N, addr);
> +	if (likely(lkey != UINT32_MAX))
> +		return lkey;
> +	/* Take slower bottom-half on miss. */
> +	return mlx5_tx_mb2mr_bh(txq, mb);
> +}
> +
> +/**
> + * Ring TX queue doorbell and flush the update if requested.
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + * @param wqe
> + *   Pointer to the last WQE posted in the NIC.
> + * @param cond
> + *   Request for write memory barrier after BlueFlame update.
> + */
> +static __rte_always_inline void
> +mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct
> mlx5_wqe *wqe,
> +		       int cond)
> +{
> +	uint64_t *dst = mlx5_tx_bfreg(txq);
> +	volatile uint64_t *src = ((volatile uint64_t *)wqe);
> +
> +	rte_io_wmb();
> +	*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
> +	/* Ensure ordering between DB record and BF copy. */
> +	rte_wmb();
> +	mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
> +	if (cond)
> +		rte_wmb();
> +}
> +
> +/**
> + * Ring TX queue doorbell and flush the update by write memory barrier.
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + * @param wqe
> + *   Pointer to the last WQE posted in the NIC.
> + */
> +static __rte_always_inline void
> +mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
> +{
> +	mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
> +}
> +
> +/**
> + * Convert timestamp from mbuf format to linear counter
> + * of Clock Queue completions (24 bits).
> + *
> + * @param sh
> + *   Pointer to the device shared context to fetch Tx
> + *   packet pacing timestamp and parameters.
> + * @param ts
> + *   Timestamp from mbuf to convert.
> + * @return
> + *   positive or zero value - completion ID to wait.
> + *   negative value - conversion error.
> + */
> +static __rte_always_inline int32_t
> +mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts)
> +{
> +	uint64_t ts, ci;
> +	uint32_t tick;
> +
> +	do {
> +		/*
> +		 * Read atomically two uint64_t fields and compare lsb bits.
> +		 * It there is no match - the timestamp was updated in
> +		 * the service thread, data should be re-read.
> +		 */
> +		rte_compiler_barrier();
> +		ci = __atomic_load_n(&sh->txpp.ts.ci_ts,
> __ATOMIC_RELAXED);
> +		ts = __atomic_load_n(&sh->txpp.ts.ts,
> __ATOMIC_RELAXED);
> +		rte_compiler_barrier();
> +		if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
> +			break;
> +	} while (true);
> +	/* Perform the skew correction, positive value to send earlier. */
> +	mts -= sh->txpp.skew;
> +	mts -= ts;
> +	if (unlikely(mts >= UINT64_MAX / 2)) {
> +		/* We have negative integer, mts is in the past. */
> +		__atomic_fetch_add(&sh->txpp.err_ts_past,
> +				   1, __ATOMIC_RELAXED);
> +		return -1;
> +	}
> +	tick = sh->txpp.tick;
> +	MLX5_ASSERT(tick);
> +	/* Convert delta to completions, round up. */
> +	mts = (mts + tick - 1) / tick;
> +	if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
> +		/* We have mts is too distant future. */
> +		__atomic_fetch_add(&sh->txpp.err_ts_future,
> +				   1, __ATOMIC_RELAXED);
> +		return -1;
> +	}
> +	mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
> +	ci += mts;
> +	ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
> +	return ci;
> +}
> +
> +#endif /* RTE_PMD_MLX5_TX_H_ */
> diff --git a/drivers/net/mlx5/mlx5_txpp.c b/drivers/net/mlx5/mlx5_txpp.c
> index 89e1c5d..d90399a 100644
> --- a/drivers/net/mlx5/mlx5_txpp.c
> +++ b/drivers/net/mlx5/mlx5_txpp.c
> @@ -16,8 +16,8 @@
>  #include <mlx5_common_devx.h>
> 
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_common_os.h"
> 
>  static_assert(sizeof(struct mlx5_cqe_ts) == sizeof(rte_int128_t),
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index cd13eb9..b8a1657 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -23,7 +23,7 @@
>  #include "mlx5_defs.h"
>  #include "mlx5_utils.h"
>  #include "mlx5.h"
> -#include "mlx5_rxtx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
> 
>  /**
> diff --git a/drivers/net/mlx5/windows/mlx5_os.c
> b/drivers/net/mlx5/windows/mlx5_os.c
> index 79eac80..814063b 100644
> --- a/drivers/net/mlx5/windows/mlx5_os.c
> +++ b/drivers/net/mlx5/windows/mlx5_os.c
> @@ -24,6 +24,7 @@
>  #include "mlx5_utils.h"
>  #include "mlx5_rxtx.h"
>  #include "mlx5_rx.h"
> +#include "mlx5_tx.h"
>  #include "mlx5_autoconf.h"
>  #include "mlx5_mr.h"
>  #include "mlx5_flow.h"
> --
> 1.8.3.1
diff mbox series

Patch

diff --git a/drivers/net/mlx5/linux/mlx5_mp_os.c b/drivers/net/mlx5/linux/mlx5_mp_os.c
index 63fa278..ca529b6 100644
--- a/drivers/net/mlx5/linux/mlx5_mp_os.c
+++ b/drivers/net/mlx5/linux/mlx5_mp_os.c
@@ -17,6 +17,7 @@ 
 #include "mlx5.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_utils.h"
 
 int
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 97a28ec..026423b 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -41,6 +41,7 @@ 
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_autoconf.h"
 #include "mlx5_mr.h"
 #include "mlx5_flow.h"
diff --git a/drivers/net/mlx5/linux/mlx5_verbs.c b/drivers/net/mlx5/linux/mlx5_verbs.c
index 73096af..0b0759f 100644
--- a/drivers/net/mlx5/linux/mlx5_verbs.c
+++ b/drivers/net/mlx5/linux/mlx5_verbs.c
@@ -20,9 +20,9 @@ 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
 #include <mlx5_common_mr.h>
-#include <mlx5_rxtx.h>
 #include <mlx5_verbs.h>
 #include <mlx5_rx.h>
+#include <mlx5_tx.h>
 #include <mlx5_utils.h>
 #include <mlx5_malloc.h>
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 6f77bc2..02cc2c7 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -36,6 +36,7 @@ 
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_autoconf.h"
 #include "mlx5_mr.h"
 #include "mlx5_flow.h"
diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
index 76935f6..76d31f5 100644
--- a/drivers/net/mlx5/mlx5_devx.c
+++ b/drivers/net/mlx5/mlx5_devx.c
@@ -20,7 +20,7 @@ 
 
 #include "mlx5.h"
 #include "mlx5_common_os.h"
-#include "mlx5_rxtx.h"
+#include "mlx5_tx.h"
 #include "mlx5_rx.h"
 #include "mlx5_utils.h"
 #include "mlx5_devx.h"
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 708e3a3..90baee5 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -24,6 +24,7 @@ 
 
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_autoconf.h"
 
 /**
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index b3877a1..4dea006 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -29,8 +29,8 @@ 
 #include "mlx5.h"
 #include "mlx5_flow.h"
 #include "mlx5_flow_os.h"
-#include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_common_os.h"
 #include "rte_pmd_mlx5.h"
 
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index cac05fb..cb5b3c9 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -32,8 +32,8 @@ 
 #include "mlx5_common_os.h"
 #include "mlx5_flow.h"
 #include "mlx5_flow_os.h"
-#include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "rte_pmd_mlx5.h"
 
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index c331350..0fdafbb 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -23,7 +23,6 @@ 
 #include "mlx5_defs.h"
 #include "mlx5.h"
 #include "mlx5_flow.h"
-#include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
 
 #define VERBS_SPEC_INNER(item_flags) \
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 2014936..e791b63 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -16,6 +16,7 @@ 
 #include "mlx5_mr.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 
 struct mr_find_contig_memsegs_data {
 	uintptr_t addr;
diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
index cf93cca..25fb47c 100644
--- a/drivers/net/mlx5/mlx5_rxmode.c
+++ b/drivers/net/mlx5/mlx5_rxmode.c
@@ -11,7 +11,6 @@ 
 
 #include <mlx5_glue.h>
 #include "mlx5.h"
-#include "mlx5_rxtx.h"
 #include "mlx5_utils.h"
 
 /**
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 19df0fa..bb9a908 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -24,7 +24,7 @@ 
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
-#include "mlx5_rxtx.h"
+#include "mlx5_tx.h"
 #include "mlx5_rx.h"
 #include "mlx5_utils.h"
 #include "mlx5_autoconf.h"
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index c7f2605..57ff407 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -26,6 +26,7 @@ 
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 
 /* TX burst subroutines return codes. */
 enum mlx5_txcmp_code {
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index f1ebc99..e168dd4 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -17,169 +17,18 @@ 
 #include <rte_spinlock.h>
 #include <rte_io.h>
 #include <rte_bus_pci.h>
-#include <rte_malloc.h>
 #include <rte_cycles.h>
 
-#include <mlx5_glue.h>
-#include <mlx5_prm.h>
 #include <mlx5_common.h>
 #include <mlx5_common_mr.h>
 
-#include "mlx5_defs.h"
 #include "mlx5_utils.h"
 #include "mlx5.h"
 #include "mlx5_autoconf.h"
 #include "mlx5_mr.h"
 
-
-/* Mbuf dynamic flag offset for inline. */
-extern uint64_t rte_net_mlx5_dynf_inline_mask;
-
-struct mlx5_txq_stats {
-#ifdef MLX5_PMD_SOFT_COUNTERS
-	uint64_t opackets; /**< Total of successfully sent packets. */
-	uint64_t obytes; /**< Total of successfully sent bytes. */
-#endif
-	uint64_t oerrors; /**< Total number of failed transmitted packets. */
-};
-
 struct mlx5_priv;
 
-/* TX queue send local data. */
-__extension__
-struct mlx5_txq_local {
-	struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */
-	struct rte_mbuf *mbuf; /* first mbuf to process. */
-	uint16_t pkts_copy; /* packets copied to elts. */
-	uint16_t pkts_sent; /* packets sent. */
-	uint16_t pkts_loop; /* packets sent on loop entry. */
-	uint16_t elts_free; /* available elts remain. */
-	uint16_t wqe_free; /* available wqe remain. */
-	uint16_t mbuf_off; /* data offset in current mbuf. */
-	uint16_t mbuf_nseg; /* number of remaining mbuf. */
-	uint16_t mbuf_free; /* number of inline mbufs to free. */
-};
-
-/* TX queue descriptor. */
-__extension__
-struct mlx5_txq_data {
-	uint16_t elts_head; /* Current counter in (*elts)[]. */
-	uint16_t elts_tail; /* Counter of first element awaiting completion. */
-	uint16_t elts_comp; /* elts index since last completion request. */
-	uint16_t elts_s; /* Number of mbuf elements. */
-	uint16_t elts_m; /* Mask for mbuf elements indices. */
-	/* Fields related to elts mbuf storage. */
-	uint16_t wqe_ci; /* Consumer index for work queue. */
-	uint16_t wqe_pi; /* Producer index for work queue. */
-	uint16_t wqe_s; /* Number of WQ elements. */
-	uint16_t wqe_m; /* Mask Number for WQ elements. */
-	uint16_t wqe_comp; /* WQE index since last completion request. */
-	uint16_t wqe_thres; /* WQE threshold to request completion in CQ. */
-	/* WQ related fields. */
-	uint16_t cq_ci; /* Consumer index for completion queue. */
-	uint16_t cq_pi; /* Production index for completion queue. */
-	uint16_t cqe_s; /* Number of CQ elements. */
-	uint16_t cqe_m; /* Mask for CQ indices. */
-	/* CQ related fields. */
-	uint16_t elts_n:4; /* elts[] length (in log2). */
-	uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
-	uint16_t wqe_n:4; /* Number of WQ elements (in log2). */
-	uint16_t tso_en:1; /* When set hardware TSO is enabled. */
-	uint16_t tunnel_en:1;
-	/* When set TX offload for tunneled packets are supported. */
-	uint16_t swp_en:1; /* Whether SW parser is enabled. */
-	uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
-	uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */
-	uint16_t db_heu:1; /* Doorbell heuristic write barrier. */
-	uint16_t fast_free:1; /* mbuf fast free on Tx is enabled. */
-	uint16_t inlen_send; /* Ordinary send data inline size. */
-	uint16_t inlen_empw; /* eMPW max packet size to inline. */
-	uint16_t inlen_mode; /* Minimal data length to inline. */
-	uint32_t qp_num_8s; /* QP number shifted by 8. */
-	uint64_t offloads; /* Offloads for Tx Queue. */
-	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
-	struct mlx5_wqe *wqes; /* Work queue. */
-	struct mlx5_wqe *wqes_end; /* Work queue array limit. */
-#ifdef RTE_LIBRTE_MLX5_DEBUG
-	uint32_t *fcqs; /* Free completion queue (debug extended). */
-#else
-	uint16_t *fcqs; /* Free completion queue. */
-#endif
-	volatile struct mlx5_cqe *cqes; /* Completion queue. */
-	volatile uint32_t *qp_db; /* Work queue doorbell. */
-	volatile uint32_t *cq_db; /* Completion queue doorbell. */
-	uint16_t port_id; /* Port ID of device. */
-	uint16_t idx; /* Queue index. */
-	uint64_t ts_mask; /* Timestamp flag dynamic mask. */
-	int32_t ts_offset; /* Timestamp field dynamic offset. */
-	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
-	struct mlx5_txq_stats stats; /* TX queue counters. */
-#ifndef RTE_ARCH_64
-	rte_spinlock_t *uar_lock;
-	/* UAR access lock required for 32bit implementations */
-#endif
-	struct rte_mbuf *elts[0];
-	/* Storage for queued packets, must be the last field. */
-} __rte_cache_aligned;
-
-enum mlx5_txq_type {
-	MLX5_TXQ_TYPE_STANDARD, /* Standard Tx queue. */
-	MLX5_TXQ_TYPE_HAIRPIN, /* Hairpin Rx queue. */
-};
-
-/* TX queue control descriptor. */
-struct mlx5_txq_ctrl {
-	LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
-	uint32_t refcnt; /* Reference counter. */
-	unsigned int socket; /* CPU socket ID for allocations. */
-	enum mlx5_txq_type type; /* The txq ctrl type. */
-	unsigned int max_inline_data; /* Max inline data. */
-	unsigned int max_tso_header; /* Max TSO header size. */
-	struct mlx5_txq_obj *obj; /* Verbs/DevX queue object. */
-	struct mlx5_priv *priv; /* Back pointer to private data. */
-	off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
-	void *bf_reg; /* BlueFlame register from Verbs. */
-	uint16_t dump_file_n; /* Number of dump files. */
-	struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
-	uint32_t hairpin_status; /* Hairpin binding status. */
-	struct mlx5_txq_data txq; /* Data path structure. */
-	/* Must be the last field in the structure, contains elts[]. */
-};
-
-#define MLX5_TX_BFREG(txq) \
-		(MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx])
-
-/* mlx5_txq.c */
-
-int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
-int mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id);
-int mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t queue_id);
-int mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t queue_id);
-int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
-			unsigned int socket, const struct rte_eth_txconf *conf);
-int mlx5_tx_hairpin_queue_setup
-	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
-	 const struct rte_eth_hairpin_conf *hairpin_conf);
-void mlx5_tx_queue_release(void *dpdk_txq);
-void txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl);
-int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd);
-void mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev);
-int mlx5_txq_obj_verify(struct rte_eth_dev *dev);
-struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
-				   uint16_t desc, unsigned int socket,
-				   const struct rte_eth_txconf *conf);
-struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
-	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
-	 const struct rte_eth_hairpin_conf *hairpin_conf);
-struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
-int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
-int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
-int mlx5_txq_verify(struct rte_eth_dev *dev);
-void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
-void txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl);
-uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
-void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
-
 /* mlx5_rxtx.c */
 
 extern uint32_t mlx5_ptype_table[];
@@ -189,88 +38,22 @@  struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
 void mlx5_set_ptype_table(void);
 void mlx5_set_cksum_table(void);
 void mlx5_set_swp_types_table(void);
-uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
-			  uint16_t pkts_n);
-int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
 void mlx5_dump_debug_information(const char *path, const char *title,
 				 const void *buf, unsigned int len);
 int mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
 			const struct mlx5_mp_arg_queue_state_modify *sm);
 int mlx5_queue_state_modify(struct rte_eth_dev *dev,
 			    struct mlx5_mp_arg_queue_state_modify *sm);
-void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
-		       struct rte_eth_txq_info *qinfo);
-int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
-			   struct rte_eth_burst_mode *mode);
 
 /* mlx5_mr.c */
 
 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
-uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb);
-uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
-			       struct rte_mempool *mp);
 int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova,
 		 size_t len);
 int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova,
 		   size_t len);
 
 /**
- * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
- * 64bit architectures.
- *
- * @param val
- *   value to write in CPU endian format.
- * @param addr
- *   Address to write to.
- * @param lock
- *   Address of the lock to use for that UAR access.
- */
-static __rte_always_inline void
-__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
-			   rte_spinlock_t *lock __rte_unused)
-{
-#ifdef RTE_ARCH_64
-	*(uint64_t *)addr = val;
-#else /* !RTE_ARCH_64 */
-	rte_spinlock_lock(lock);
-	*(uint32_t *)addr = val;
-	rte_io_wmb();
-	*((uint32_t *)addr + 1) = val >> 32;
-	rte_spinlock_unlock(lock);
-#endif
-}
-
-/**
- * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
- * 64bit architectures while guaranteeing the order of execution with the
- * code being executed.
- *
- * @param val
- *   value to write in CPU endian format.
- * @param addr
- *   Address to write to.
- * @param lock
- *   Address of the lock to use for that UAR access.
- */
-static __rte_always_inline void
-__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
-{
-	rte_io_wmb();
-	__mlx5_uar_write64_relaxed(val, addr, lock);
-}
-
-/* Assist macros, used instead of directly calling the functions they wrap. */
-#ifdef RTE_ARCH_64
-#define mlx5_uar_write64_relaxed(val, dst, lock) \
-		__mlx5_uar_write64_relaxed(val, dst, NULL)
-#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL)
-#else
-#define mlx5_uar_write64_relaxed(val, dst, lock) \
-		__mlx5_uar_write64_relaxed(val, dst, lock)
-#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
-#endif
-
-/**
  * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
  * cloned mbuf is allocated is returned instead.
  *
@@ -288,131 +71,4 @@  int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova,
 	return buf->pool;
 }
 
-/**
- * Query LKey from a packet buffer for Tx. If not found, add the mempool.
- *
- * @param txq
- *   Pointer to Tx queue structure.
- * @param addr
- *   Address to search.
- *
- * @return
- *   Searched LKey on success, UINT32_MAX on no match.
- */
-static __rte_always_inline uint32_t
-mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
-{
-	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
-	uintptr_t addr = (uintptr_t)mb->buf_addr;
-	uint32_t lkey;
-
-	/* Check generation bit to see if there's any change on existing MRs. */
-	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
-		mlx5_mr_flush_local_cache(mr_ctrl);
-	/* Linear search on MR cache array. */
-	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
-				   MLX5_MR_CACHE_N, addr);
-	if (likely(lkey != UINT32_MAX))
-		return lkey;
-	/* Take slower bottom-half on miss. */
-	return mlx5_tx_mb2mr_bh(txq, mb);
-}
-
-/**
- * Ring TX queue doorbell and flush the update if requested.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param wqe
- *   Pointer to the last WQE posted in the NIC.
- * @param cond
- *   Request for write memory barrier after BlueFlame update.
- */
-static __rte_always_inline void
-mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
-		       int cond)
-{
-	uint64_t *dst = MLX5_TX_BFREG(txq);
-	volatile uint64_t *src = ((volatile uint64_t *)wqe);
-
-	rte_io_wmb();
-	*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
-	/* Ensure ordering between DB record and BF copy. */
-	rte_wmb();
-	mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
-	if (cond)
-		rte_wmb();
-}
-
-/**
- * Ring TX queue doorbell and flush the update by write memory barrier.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param wqe
- *   Pointer to the last WQE posted in the NIC.
- */
-static __rte_always_inline void
-mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
-{
-	mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
-}
-
-/**
- * Convert timestamp from mbuf format to linear counter
- * of Clock Queue completions (24 bits)
- *
- * @param sh
- *   Pointer to the device shared context to fetch Tx
- *   packet pacing timestamp and parameters.
- * @param ts
- *   Timestamp from mbuf to convert.
- * @return
- *   positive or zero value - completion ID to wait
- *   negative value - conversion error
- */
-static __rte_always_inline int32_t
-mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts)
-{
-	uint64_t ts, ci;
-	uint32_t tick;
-
-	do {
-		/*
-		 * Read atomically two uint64_t fields and compare lsb bits.
-		 * It there is no match - the timestamp was updated in
-		 * the service thread, data should be re-read.
-		 */
-		rte_compiler_barrier();
-		ci = __atomic_load_n(&sh->txpp.ts.ci_ts, __ATOMIC_RELAXED);
-		ts = __atomic_load_n(&sh->txpp.ts.ts, __ATOMIC_RELAXED);
-		rte_compiler_barrier();
-		if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
-			break;
-	} while (true);
-	/* Perform the skew correction, positive value to send earlier. */
-	mts -= sh->txpp.skew;
-	mts -= ts;
-	if (unlikely(mts >= UINT64_MAX / 2)) {
-		/* We have negative integer, mts is in the past. */
-		__atomic_fetch_add(&sh->txpp.err_ts_past,
-				   1, __ATOMIC_RELAXED);
-		return -1;
-	}
-	tick = sh->txpp.tick;
-	MLX5_ASSERT(tick);
-	/* Convert delta to completions, round up. */
-	mts = (mts + tick - 1) / tick;
-	if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
-		/* We have mts is too distant future. */
-		__atomic_fetch_add(&sh->txpp.err_ts_future,
-				   1, __ATOMIC_RELAXED);
-		return -1;
-	}
-	mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
-	ci += mts;
-	ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
-	return ci;
-}
-
 #endif /* RTE_PMD_MLX5_RXTX_H_ */
diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c
index 4dbd831..ae2f566 100644
--- a/drivers/net/mlx5/mlx5_stats.c
+++ b/drivers/net/mlx5/mlx5_stats.c
@@ -16,8 +16,8 @@ 
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
-#include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_malloc.h"
 
 /**
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index c88cb22..001c0b5 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -15,8 +15,8 @@ 
 
 #include "mlx5.h"
 #include "mlx5_mr.h"
-#include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_utils.h"
 #include "rte_pmd_mlx5.h"
 
diff --git a/drivers/net/mlx5/mlx5_tx.h b/drivers/net/mlx5/mlx5_tx.h
new file mode 100644
index 0000000..7f91d04
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_tx.h
@@ -0,0 +1,371 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 6WIND S.A.
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_TX_H_
+#define RTE_PMD_MLX5_TX_H_
+
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_common.h>
+#include <rte_spinlock.h>
+
+#include <mlx5_common_mr.h>
+
+#include "mlx5.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_mr.h"
+
+/* Mbuf dynamic flag offset for inline. */
+extern uint64_t rte_net_mlx5_dynf_inline_mask;
+
+struct mlx5_txq_stats {
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint64_t opackets; /**< Total of successfully sent packets. */
+	uint64_t obytes; /**< Total of successfully sent bytes. */
+#endif
+	uint64_t oerrors; /**< Total number of failed transmitted packets. */
+};
+
+/* TX queue send local data. */
+__extension__
+struct mlx5_txq_local {
+	struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */
+	struct rte_mbuf *mbuf; /* first mbuf to process. */
+	uint16_t pkts_copy; /* packets copied to elts. */
+	uint16_t pkts_sent; /* packets sent. */
+	uint16_t pkts_loop; /* packets sent on loop entry. */
+	uint16_t elts_free; /* available elts remain. */
+	uint16_t wqe_free; /* available wqe remain. */
+	uint16_t mbuf_off; /* data offset in current mbuf. */
+	uint16_t mbuf_nseg; /* number of remaining mbuf. */
+	uint16_t mbuf_free; /* number of inline mbufs to free. */
+};
+
+/* TX queue descriptor. */
+__extension__
+struct mlx5_txq_data {
+	uint16_t elts_head; /* Current counter in (*elts)[]. */
+	uint16_t elts_tail; /* Counter of first element awaiting completion. */
+	uint16_t elts_comp; /* elts index since last completion request. */
+	uint16_t elts_s; /* Number of mbuf elements. */
+	uint16_t elts_m; /* Mask for mbuf elements indices. */
+	/* Fields related to elts mbuf storage. */
+	uint16_t wqe_ci; /* Consumer index for work queue. */
+	uint16_t wqe_pi; /* Producer index for work queue. */
+	uint16_t wqe_s; /* Number of WQ elements. */
+	uint16_t wqe_m; /* Mask Number for WQ elements. */
+	uint16_t wqe_comp; /* WQE index since last completion request. */
+	uint16_t wqe_thres; /* WQE threshold to request completion in CQ. */
+	/* WQ related fields. */
+	uint16_t cq_ci; /* Consumer index for completion queue. */
+	uint16_t cq_pi; /* Production index for completion queue. */
+	uint16_t cqe_s; /* Number of CQ elements. */
+	uint16_t cqe_m; /* Mask for CQ indices. */
+	/* CQ related fields. */
+	uint16_t elts_n:4; /* elts[] length (in log2). */
+	uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
+	uint16_t wqe_n:4; /* Number of WQ elements (in log2). */
+	uint16_t tso_en:1; /* When set hardware TSO is enabled. */
+	uint16_t tunnel_en:1;
+	/* When set TX offload for tunneled packets are supported. */
+	uint16_t swp_en:1; /* Whether SW parser is enabled. */
+	uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
+	uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */
+	uint16_t db_heu:1; /* Doorbell heuristic write barrier. */
+	uint16_t fast_free:1; /* mbuf fast free on Tx is enabled. */
+	uint16_t inlen_send; /* Ordinary send data inline size. */
+	uint16_t inlen_empw; /* eMPW max packet size to inline. */
+	uint16_t inlen_mode; /* Minimal data length to inline. */
+	uint32_t qp_num_8s; /* QP number shifted by 8. */
+	uint64_t offloads; /* Offloads for Tx Queue. */
+	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
+	struct mlx5_wqe *wqes; /* Work queue. */
+	struct mlx5_wqe *wqes_end; /* Work queue array limit. */
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+	uint32_t *fcqs; /* Free completion queue (debug extended). */
+#else
+	uint16_t *fcqs; /* Free completion queue. */
+#endif
+	volatile struct mlx5_cqe *cqes; /* Completion queue. */
+	volatile uint32_t *qp_db; /* Work queue doorbell. */
+	volatile uint32_t *cq_db; /* Completion queue doorbell. */
+	uint16_t port_id; /* Port ID of device. */
+	uint16_t idx; /* Queue index. */
+	uint64_t ts_mask; /* Timestamp flag dynamic mask. */
+	int32_t ts_offset; /* Timestamp field dynamic offset. */
+	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
+	struct mlx5_txq_stats stats; /* TX queue counters. */
+#ifndef RTE_ARCH_64
+	rte_spinlock_t *uar_lock;
+	/* UAR access lock required for 32bit implementations */
+#endif
+	struct rte_mbuf *elts[0];
+	/* Storage for queued packets, must be the last field. */
+} __rte_cache_aligned;
+
+enum mlx5_txq_type {
+	MLX5_TXQ_TYPE_STANDARD, /* Standard Tx queue. */
+	MLX5_TXQ_TYPE_HAIRPIN, /* Hairpin Tx queue. */
+};
+
+/* TX queue control descriptor. */
+struct mlx5_txq_ctrl {
+	LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
+	uint32_t refcnt; /* Reference counter. */
+	unsigned int socket; /* CPU socket ID for allocations. */
+	enum mlx5_txq_type type; /* The txq ctrl type. */
+	unsigned int max_inline_data; /* Max inline data. */
+	unsigned int max_tso_header; /* Max TSO header size. */
+	struct mlx5_txq_obj *obj; /* Verbs/DevX queue object. */
+	struct mlx5_priv *priv; /* Back pointer to private data. */
+	off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
+	void *bf_reg; /* BlueFlame register from Verbs. */
+	uint16_t dump_file_n; /* Number of dump files. */
+	struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
+	uint32_t hairpin_status; /* Hairpin binding status. */
+	struct mlx5_txq_data txq; /* Data path structure. */
+	/* Must be the last field in the structure, contains elts[]. */
+};
+
+/* mlx5_txq.c */
+
+int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
+int mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id);
+int mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t queue_id);
+int mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t queue_id);
+int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+			unsigned int socket, const struct rte_eth_txconf *conf);
+int mlx5_tx_hairpin_queue_setup
+	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+	 const struct rte_eth_hairpin_conf *hairpin_conf);
+void mlx5_tx_queue_release(void *dpdk_txq);
+void txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl);
+int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd);
+void mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev);
+int mlx5_txq_obj_verify(struct rte_eth_dev *dev);
+struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
+				   uint16_t desc, unsigned int socket,
+				   const struct rte_eth_txconf *conf);
+struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
+	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+	 const struct rte_eth_hairpin_conf *hairpin_conf);
+struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_txq_verify(struct rte_eth_dev *dev);
+void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
+void txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl);
+uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
+void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
+
+/* mlx5_tx.c */
+
+uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
+			  uint16_t pkts_n);
+int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
+void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+		       struct rte_eth_txq_info *qinfo);
+int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+			   struct rte_eth_burst_mode *mode);
+
+/* mlx5_mr.c */
+
+uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb);
+uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
+			       struct rte_mempool *mp);
+
+static __rte_always_inline uint64_t *
+mlx5_tx_bfreg(struct mlx5_txq_data *txq)
+{
+	return MLX5_PROC_PRIV(txq->port_id)->uar_table[txq->idx];
+}
+
+/**
+ * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
+ * 64bit architectures.
+ *
+ * @param val
+ *   value to write in CPU endian format.
+ * @param addr
+ *   Address to write to.
+ * @param lock
+ *   Address of the lock to use for that UAR access.
+ */
+static __rte_always_inline void
+__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
+			   rte_spinlock_t *lock __rte_unused)
+{
+#ifdef RTE_ARCH_64
+	*(uint64_t *)addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(lock);
+	*(uint32_t *)addr = val;
+	rte_io_wmb();
+	*((uint32_t *)addr + 1) = val >> 32;
+	rte_spinlock_unlock(lock);
+#endif
+}
+
+/**
+ * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
+ * 64bit architectures while guaranteeing the order of execution with the
+ * code being executed.
+ *
+ * @param val
+ *   value to write in CPU endian format.
+ * @param addr
+ *   Address to write to.
+ * @param lock
+ *   Address of the lock to use for that UAR access.
+ */
+static __rte_always_inline void
+__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
+{
+	rte_io_wmb();
+	__mlx5_uar_write64_relaxed(val, addr, lock);
+}
+
+/* Assist macros, used instead of directly calling the functions they wrap. */
+#ifdef RTE_ARCH_64
+#define mlx5_uar_write64_relaxed(val, dst, lock) \
+		__mlx5_uar_write64_relaxed(val, dst, NULL)
+#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL)
+#else
+#define mlx5_uar_write64_relaxed(val, dst, lock) \
+		__mlx5_uar_write64_relaxed(val, dst, lock)
+#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
+#endif
+
+/**
+ * Query LKey from a packet buffer for Tx. If not found, add the mempool.
+ *
+ * @param txq
+ *   Pointer to Tx queue structure.
+ * @param addr
+ *   Address to search.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
+{
+	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+	uintptr_t addr = (uintptr_t)mb->buf_addr;
+	uint32_t lkey;
+
+	/* Check generation bit to see if there's any change on existing MRs. */
+	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+		mlx5_mr_flush_local_cache(mr_ctrl);
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+				   MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half on miss. */
+	return mlx5_tx_mb2mr_bh(txq, mb);
+}
+
+/**
+ * Ring TX queue doorbell and flush the update if requested.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param wqe
+ *   Pointer to the last WQE posted in the NIC.
+ * @param cond
+ *   Request for write memory barrier after BlueFlame update.
+ */
+static __rte_always_inline void
+mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
+		       int cond)
+{
+	uint64_t *dst = mlx5_tx_bfreg(txq);
+	volatile uint64_t *src = ((volatile uint64_t *)wqe);
+
+	rte_io_wmb();
+	*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
+	/* Ensure ordering between DB record and BF copy. */
+	rte_wmb();
+	mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
+	if (cond)
+		rte_wmb();
+}
+
+/**
+ * Ring TX queue doorbell and flush the update by write memory barrier.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param wqe
+ *   Pointer to the last WQE posted in the NIC.
+ */
+static __rte_always_inline void
+mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
+{
+	mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
+}
+
+/**
+ * Convert timestamp from mbuf format to linear counter
+ * of Clock Queue completions (24 bits).
+ *
+ * @param sh
+ *   Pointer to the device shared context to fetch Tx
+ *   packet pacing timestamp and parameters.
+ * @param ts
+ *   Timestamp from mbuf to convert.
+ * @return
+ *   positive or zero value - completion ID to wait.
+ *   negative value - conversion error.
+ */
+static __rte_always_inline int32_t
+mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts)
+{
+	uint64_t ts, ci;
+	uint32_t tick;
+
+	do {
+		/*
+		 * Read atomically two uint64_t fields and compare lsb bits.
+		 * It there is no match - the timestamp was updated in
+		 * the service thread, data should be re-read.
+		 */
+		rte_compiler_barrier();
+		ci = __atomic_load_n(&sh->txpp.ts.ci_ts, __ATOMIC_RELAXED);
+		ts = __atomic_load_n(&sh->txpp.ts.ts, __ATOMIC_RELAXED);
+		rte_compiler_barrier();
+		if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
+			break;
+	} while (true);
+	/* Perform the skew correction, positive value to send earlier. */
+	mts -= sh->txpp.skew;
+	mts -= ts;
+	if (unlikely(mts >= UINT64_MAX / 2)) {
+		/* We have negative integer, mts is in the past. */
+		__atomic_fetch_add(&sh->txpp.err_ts_past,
+				   1, __ATOMIC_RELAXED);
+		return -1;
+	}
+	tick = sh->txpp.tick;
+	MLX5_ASSERT(tick);
+	/* Convert delta to completions, round up. */
+	mts = (mts + tick - 1) / tick;
+	if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
+		/* We have mts is too distant future. */
+		__atomic_fetch_add(&sh->txpp.err_ts_future,
+				   1, __ATOMIC_RELAXED);
+		return -1;
+	}
+	mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
+	ci += mts;
+	ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
+	return ci;
+}
+
+#endif /* RTE_PMD_MLX5_TX_H_ */
diff --git a/drivers/net/mlx5/mlx5_txpp.c b/drivers/net/mlx5/mlx5_txpp.c
index 89e1c5d..d90399a 100644
--- a/drivers/net/mlx5/mlx5_txpp.c
+++ b/drivers/net/mlx5/mlx5_txpp.c
@@ -16,8 +16,8 @@ 
 #include <mlx5_common_devx.h>
 
 #include "mlx5.h"
-#include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_common_os.h"
 
 static_assert(sizeof(struct mlx5_cqe_ts) == sizeof(rte_int128_t),
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index cd13eb9..b8a1657 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -23,7 +23,7 @@ 
 #include "mlx5_defs.h"
 #include "mlx5_utils.h"
 #include "mlx5.h"
-#include "mlx5_rxtx.h"
+#include "mlx5_tx.h"
 #include "mlx5_autoconf.h"
 
 /**
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 79eac80..814063b 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -24,6 +24,7 @@ 
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"
+#include "mlx5_tx.h"
 #include "mlx5_autoconf.h"
 #include "mlx5_mr.h"
 #include "mlx5_flow.h"