Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 56457,
    "url": "http://patches.dpdk.org/api/patches/56457/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/1563199161-29745-7-git-send-email-viacheslavo@mellanox.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<1563199161-29745-7-git-send-email-viacheslavo@mellanox.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/1563199161-29745-7-git-send-email-viacheslavo@mellanox.com",
    "date": "2019-07-15T13:59:20",
    "name": "[v2,6/7] net/mlx5: implement Tx burst template",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "4b9a4eea7d2af186c06ef4f221524d1a33f67abf",
    "submitter": {
        "id": 1102,
        "url": "http://patches.dpdk.org/api/people/1102/?format=api",
        "name": "Slava Ovsiienko",
        "email": "viacheslavo@mellanox.com"
    },
    "delegate": {
        "id": 3268,
        "url": "http://patches.dpdk.org/api/users/3268/?format=api",
        "username": "rasland",
        "first_name": "Raslan",
        "last_name": "Darawsheh",
        "email": "rasland@nvidia.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/1563199161-29745-7-git-send-email-viacheslavo@mellanox.com/mbox/",
    "series": [
        {
            "id": 5500,
            "url": "http://patches.dpdk.org/api/series/5500/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=5500",
            "date": "2019-07-15T13:59:14",
            "name": "net/mlx5: consolidate Tx datapath",
            "version": 2,
            "mbox": "http://patches.dpdk.org/series/5500/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/56457/comments/",
    "check": "success",
    "checks": "http://patches.dpdk.org/api/patches/56457/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@dpdk.org",
        "Delivered-To": "patchwork@dpdk.org",
        "Received": [
            "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 0BEB51BA5E;\n\tMon, 15 Jul 2019 16:00:05 +0200 (CEST)",
            "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n\tby dpdk.org (Postfix) with ESMTP id D9E461BD4F\n\tfor <dev@dpdk.org>; Mon, 15 Jul 2019 16:00:02 +0200 (CEST)",
            "from Internal Mail-Server by MTLPINE2 (envelope-from\n\tviacheslavo@mellanox.com)\n\twith ESMTPS (AES256-SHA encrypted); 15 Jul 2019 16:59:59 +0300",
            "from pegasus12.mtr.labs.mlnx. (pegasus12.mtr.labs.mlnx\n\t[10.210.17.40])\n\tby labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x6FDxOEP013758;\n\tMon, 15 Jul 2019 16:59:58 +0300"
        ],
        "From": "Viacheslav Ovsiienko <viacheslavo@mellanox.com>",
        "To": "dev@dpdk.org",
        "Cc": "yskoh@mellanox.com",
        "Date": "Mon, 15 Jul 2019 13:59:20 +0000",
        "Message-Id": "<1563199161-29745-7-git-send-email-viacheslavo@mellanox.com>",
        "X-Mailer": "git-send-email 1.8.3.1",
        "In-Reply-To": "<1563199161-29745-1-git-send-email-viacheslavo@mellanox.com>",
        "References": "<1562257767-19035-2-git-send-email-viacheslavo@mellanox.com>\n\t<1563199161-29745-1-git-send-email-viacheslavo@mellanox.com>",
        "Subject": "[dpdk-dev] [PATCH v2 6/7] net/mlx5: implement Tx burst template",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.15",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "This patch adds the implementation of tx_burst routine template.\nThe template supports all Tx offloads and multiple optimized\ntx_burst routines can be generated by compiler from this one.\n\nSigned-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>\n---\n drivers/net/mlx5/mlx5_rxtx.c | 2897 +++++++++++++++++++++++++++++++++++++++++-\n drivers/net/mlx5/mlx5_rxtx.h |    5 +-\n 2 files changed, 2873 insertions(+), 29 deletions(-)",
    "diff": "diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c\nindex af6f705..2674ea8 100644\n--- a/drivers/net/mlx5/mlx5_rxtx.c\n+++ b/drivers/net/mlx5/mlx5_rxtx.c\n@@ -339,6 +339,109 @@ enum mlx5_txcmp_code {\n }\n \n /**\n+ * Set Software Parser flags and offsets in Ethernet Segment of WQE.\n+ * Flags must be preliminary initialized to zero.\n+ *\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param swp_flags\n+ *   Pointer to store Software Parser flags\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   Software Parser offsets packed in dword.\n+ *   Software Parser flags are set by pointer.\n+ */\n+static __rte_always_inline uint32_t\n+txq_mbuf_to_swp(struct mlx5_txq_local *restrict loc,\n+\t\tuint8_t *swp_flags,\n+\t\tunsigned int olx)\n+{\n+\tuint64_t ol, tunnel;\n+\tunsigned int idx, off;\n+\tuint32_t set;\n+\n+\tif (!MLX5_TXOFF_CONFIG(SWP))\n+\t\treturn 0;\n+\tol = loc->mbuf->ol_flags;\n+\ttunnel = ol & PKT_TX_TUNNEL_MASK;\n+\t/*\n+\t * Check whether Software Parser is required.\n+\t * Only customized tunnels may ask for.\n+\t */\n+\tif (likely(tunnel != PKT_TX_TUNNEL_UDP && tunnel != PKT_TX_TUNNEL_IP))\n+\t\treturn 0;\n+\t/*\n+\t * The index should have:\n+\t * bit[0:1] = PKT_TX_L4_MASK\n+\t * bit[4] = PKT_TX_IPV6\n+\t * bit[8] = PKT_TX_OUTER_IPV6\n+\t * bit[9] = PKT_TX_OUTER_UDP\n+\t */\n+\tidx = (ol & (PKT_TX_L4_MASK | PKT_TX_IPV6 | PKT_TX_OUTER_IPV6)) >> 52;\n+\tidx |= (tunnel == PKT_TX_TUNNEL_UDP) ? (1 << 9) : 0;\n+\t*swp_flags = mlx5_swp_types_table[idx];\n+\t/*\n+\t * Set offsets for SW parser. Since ConnectX-5, SW parser just\n+\t * complements HW parser. SW parser starts to engage only if HW parser\n+\t * can't reach a header. For the older devices, HW parser will not kick\n+\t * in if any of SWP offsets is set. Therefore, all of the L3 offsets\n+\t * should be set regardless of HW offload.\n+\t */\n+\toff = loc->mbuf->outer_l2_len;\n+\tif (MLX5_TXOFF_CONFIG(VLAN) && ol & PKT_TX_VLAN_PKT)\n+\t\toff += sizeof(struct rte_vlan_hdr);\n+\tset = (off >> 1) << 8; /* Outer L3 offset. */\n+\toff += loc->mbuf->outer_l3_len;\n+\tif (tunnel == PKT_TX_TUNNEL_UDP)\n+\t\tset |= off >> 1; /* Outer L4 offset. */\n+\tif (ol & (PKT_TX_IPV4 | PKT_TX_IPV6)) { /* Inner IP. */\n+\t\tconst uint64_t csum = ol & PKT_TX_L4_MASK;\n+\t\t\toff += loc->mbuf->l2_len;\n+\t\tset |= (off >> 1) << 24; /* Inner L3 offset. */\n+\t\tif (csum == PKT_TX_TCP_CKSUM ||\n+\t\t    csum == PKT_TX_UDP_CKSUM ||\n+\t\t    (MLX5_TXOFF_CONFIG(TSO) && ol & PKT_TX_TCP_SEG)) {\n+\t\t\toff += loc->mbuf->l3_len;\n+\t\t\tset |= (off >> 1) << 16; /* Inner L4 offset. */\n+\t\t}\n+\t}\n+\tset = rte_cpu_to_le_32(set);\n+\treturn set;\n+}\n+\n+/**\n+ * Convert the Checksum offloads to Verbs.\n+ *\n+ * @param buf\n+ *   Pointer to the mbuf.\n+ *\n+ * @return\n+ *   Converted checksum flags.\n+ */\n+static __rte_always_inline uint8_t\n+txq_ol_cksum_to_cs(struct rte_mbuf *buf)\n+{\n+\tuint32_t idx;\n+\tuint8_t is_tunnel = !!(buf->ol_flags & PKT_TX_TUNNEL_MASK);\n+\tconst uint64_t ol_flags_mask = PKT_TX_TCP_SEG | PKT_TX_L4_MASK |\n+\t\t\t\t       PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM;\n+\n+\t/*\n+\t * The index should have:\n+\t * bit[0] = PKT_TX_TCP_SEG\n+\t * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM\n+\t * bit[4] = PKT_TX_IP_CKSUM\n+\t * bit[8] = PKT_TX_OUTER_IP_CKSUM\n+\t * bit[9] = tunnel\n+\t */\n+\tidx = ((buf->ol_flags & ol_flags_mask) >> 50) | (!!is_tunnel << 9);\n+\treturn mlx5_cksum_table[idx];\n+}\n+\n+/**\n  * Internal function to compute the number of used descriptors in an RX queue\n  *\n  * @param rxq\n@@ -543,7 +646,7 @@ enum mlx5_txcmp_code {\n  *   The last Tx buffer element to free.\n  */\n uint16_t\n-mlx5_tx_error_cqe_handle(struct mlx5_txq_data *txq,\n+mlx5_tx_error_cqe_handle(struct mlx5_txq_data *restrict txq,\n \t\t\t volatile struct mlx5_err_cqe *err_cqe)\n {\n \tif (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) {\n@@ -1563,6 +1666,298 @@ enum mlx5_txcmp_code {\n }\n \n /**\n+ * Free the mbufs from the linear array of pointers.\n+ *\n+ * @param pkts\n+ *   Pointer to array of packets to be free.\n+ * @param pkts_n\n+ *   Number of packets to be freed.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_free_mbuf(struct rte_mbuf **restrict pkts,\n+\t\t  unsigned int pkts_n,\n+\t\t  unsigned int olx __rte_unused)\n+{\n+\tstruct rte_mempool *pool = NULL;\n+\tstruct rte_mbuf **p_free = NULL;\n+\tstruct rte_mbuf *mbuf;\n+\tunsigned int n_free = 0;\n+\n+\t/*\n+\t * The implemented algorithm eliminates\n+\t * copying pointers to temporary array\n+\t * for rte_mempool_put_bulk() calls.\n+\t */\n+\tassert(pkts);\n+\tassert(pkts_n);\n+\tfor (;;) {\n+\t\tfor (;;) {\n+\t\t\t/*\n+\t\t\t * Decrement mbuf reference counter, detach\n+\t\t\t * indirect and external buffers if needed.\n+\t\t\t */\n+\t\t\tmbuf = rte_pktmbuf_prefree_seg(*pkts);\n+\t\t\tif (likely(mbuf != NULL)) {\n+\t\t\t\tassert(mbuf == *pkts);\n+\t\t\t\tif (likely(n_free != 0)) {\n+\t\t\t\t\tif (unlikely(pool != mbuf->pool))\n+\t\t\t\t\t\t/* From different pool. */\n+\t\t\t\t\t\tbreak;\n+\t\t\t\t} else {\n+\t\t\t\t\t/* Start new scan array. */\n+\t\t\t\t\tpool = mbuf->pool;\n+\t\t\t\t\tp_free = pkts;\n+\t\t\t\t}\n+\t\t\t\t++n_free;\n+\t\t\t\t++pkts;\n+\t\t\t\t--pkts_n;\n+\t\t\t\tif (unlikely(pkts_n == 0)) {\n+\t\t\t\t\tmbuf = NULL;\n+\t\t\t\t\tbreak;\n+\t\t\t\t}\n+\t\t\t} else {\n+\t\t\t\t/*\n+\t\t\t\t * This happens if mbuf is still referenced.\n+\t\t\t\t * We can't put it back to the pool, skip.\n+\t\t\t\t */\n+\t\t\t\t++pkts;\n+\t\t\t\t--pkts_n;\n+\t\t\t\tif (unlikely(n_free != 0))\n+\t\t\t\t\t/* There is some array to free.*/\n+\t\t\t\t\tbreak;\n+\t\t\t\tif (unlikely(pkts_n == 0))\n+\t\t\t\t\t/* Last mbuf, nothing to free. */\n+\t\t\t\t\treturn;\n+\t\t\t}\n+\t\t}\n+\t\tfor (;;) {\n+\t\t\t/*\n+\t\t\t * This loop is implemented to avoid multiple\n+\t\t\t * inlining of rte_mempool_put_bulk().\n+\t\t\t */\n+\t\t\tassert(pool);\n+\t\t\tassert(p_free);\n+\t\t\tassert(n_free);\n+\t\t\t/*\n+\t\t\t * Free the array of pre-freed mbufs\n+\t\t\t * belonging to the same memory pool.\n+\t\t\t */\n+\t\t\trte_mempool_put_bulk(pool, (void *)p_free, n_free);\n+\t\t\tif (unlikely(mbuf != NULL)) {\n+\t\t\t\t/* There is the request to start new scan. */\n+\t\t\t\tpool = mbuf->pool;\n+\t\t\t\tp_free = pkts++;\n+\t\t\t\tn_free = 1;\n+\t\t\t\t--pkts_n;\n+\t\t\t\tif (likely(pkts_n != 0))\n+\t\t\t\t\tbreak;\n+\t\t\t\t/*\n+\t\t\t\t * This is the last mbuf to be freed.\n+\t\t\t\t * Do one more loop iteration to complete.\n+\t\t\t\t * This is rare case of the last unique mbuf.\n+\t\t\t\t */\n+\t\t\t\tmbuf = NULL;\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tif (likely(pkts_n == 0))\n+\t\t\t\treturn;\n+\t\t\tn_free = 0;\n+\t\t\tbreak;\n+\t\t}\n+\t}\n+}\n+\n+/**\n+ * Free the mbuf from the elts ring buffer till new tail.\n+ *\n+ * @param txq\n+ *   Pointer to Tx queue structure.\n+ * @param tail\n+ *   Index in elts to free up to, becomes new elts tail.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_free_elts(struct mlx5_txq_data *restrict txq,\n+\t\t  uint16_t tail,\n+\t\t  unsigned int olx __rte_unused)\n+{\n+\tuint16_t n_elts = tail - txq->elts_tail;\n+\n+\tassert(n_elts);\n+\tassert(n_elts <= txq->elts_s);\n+\t/*\n+\t * Implement a loop to support ring buffer wraparound\n+\t * with single inlining of mlx5_tx_free_mbuf().\n+\t */\n+\tdo {\n+\t\tunsigned int part;\n+\n+\t\tpart = txq->elts_s - (txq->elts_tail & txq->elts_m);\n+\t\tpart = RTE_MIN(part, n_elts);\n+\t\tassert(part);\n+\t\tassert(part <= txq->elts_s);\n+\t\tmlx5_tx_free_mbuf(&txq->elts[txq->elts_tail & txq->elts_m],\n+\t\t\t\t  part, olx);\n+\t\ttxq->elts_tail += part;\n+\t\tn_elts -= part;\n+\t} while (n_elts);\n+}\n+\n+/**\n+ * Store the mbuf being sent into elts ring buffer.\n+ * On Tx completion these mbufs will be freed.\n+ *\n+ * @param txq\n+ *   Pointer to Tx queue structure.\n+ * @param pkts\n+ *   Pointer to array of packets to be stored.\n+ * @param pkts_n\n+ *   Number of packets to be stored.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_copy_elts(struct mlx5_txq_data *restrict txq,\n+\t\t  struct rte_mbuf **restrict pkts,\n+\t\t  unsigned int pkts_n,\n+\t\t  unsigned int olx __rte_unused)\n+{\n+\tunsigned int part;\n+\tstruct rte_mbuf **elts = (struct rte_mbuf **)txq->elts;\n+\n+\tassert(pkts);\n+\tassert(pkts_n);\n+\tpart = txq->elts_s - (txq->elts_head & txq->elts_m);\n+\tassert(part);\n+\tassert(part <= txq->elts_s);\n+\t/* This code is a good candidate for vectorizing with SIMD. */\n+\trte_memcpy((void *)(elts + (txq->elts_head & txq->elts_m)),\n+\t\t   (void *)pkts,\n+\t\t   RTE_MIN(part, pkts_n) * sizeof(struct rte_mbuf *));\n+\ttxq->elts_head += pkts_n;\n+\tif (unlikely(part < pkts_n))\n+\t\t/* The copy is wrapping around the elts array. */\n+\t\trte_memcpy((void *)elts, (void *)(pkts + part),\n+\t\t\t   (pkts_n - part) * sizeof(struct rte_mbuf *));\n+}\n+\n+/**\n+ * Manage TX completions. This routine checks the CQ for\n+ * arrived CQEs, deduces the last accomplished WQE in SQ,\n+ * updates SQ producing index and frees all completed mbufs.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * NOTE: not inlined intentionally, it makes tx_burst\n+ * routine smaller, simple and faster - from experiments.\n+ */\n+static void\n+mlx5_tx_handle_completion(struct mlx5_txq_data *restrict txq,\n+\t\t\t  unsigned int olx __rte_unused)\n+{\n+\tbool update = false;\n+\tint ret;\n+\n+\tdo {\n+\t\tvolatile struct mlx5_wqe_cseg *cseg;\n+\t\tvolatile struct mlx5_cqe *cqe;\n+\t\tuint16_t tail;\n+\n+\t\tcqe = &txq->cqes[txq->cq_ci & txq->cqe_m];\n+\t\tret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);\n+\t\tif (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {\n+\t\t\tif (likely(ret != MLX5_CQE_STATUS_ERR)) {\n+\t\t\t\t/* No new CQEs in completion queue. */\n+\t\t\t\tassert(ret == MLX5_CQE_STATUS_HW_OWN);\n+\t\t\t\tif (likely(update)) {\n+\t\t\t\t\t/* Update the consumer index. */\n+\t\t\t\t\trte_compiler_barrier();\n+\t\t\t\t\t*txq->cq_db =\n+\t\t\t\t\t\trte_cpu_to_be_32(txq->cq_ci);\n+\t\t\t\t}\n+\t\t\t\treturn;\n+\t\t\t}\n+\t\t\t/* Some error occurred, try to restart. */\n+\t\t\trte_wmb();\n+\t\t\ttail = mlx5_tx_error_cqe_handle\n+\t\t\t\t(txq, (volatile struct mlx5_err_cqe *)cqe);\n+\t\t} else {\n+\t\t\t/* Normal transmit completion. */\n+\t\t\t++txq->cq_ci;\n+\t\t\trte_cio_rmb();\n+\t\t\ttxq->wqe_pi = rte_be_to_cpu_16(cqe->wqe_counter);\n+\t\t\tcseg = (volatile struct mlx5_wqe_cseg *)\n+\t\t\t\t(txq->wqes + (txq->wqe_pi & txq->wqe_m));\n+\t\t\ttail = cseg->misc;\n+\t\t}\n+#ifndef NDEBUG\n+\t\tif (txq->cq_pi)\n+\t\t\t--txq->cq_pi;\n+#endif\n+\t\tif (likely(tail != txq->elts_tail)) {\n+\t\t\t/* Free data buffers from elts. */\n+\t\t\tmlx5_tx_free_elts(txq, tail, olx);\n+\t\t\tassert(tail == txq->elts_tail);\n+\t\t}\n+\t\tupdate = true;\n+\t} while (true);\n+}\n+\n+/**\n+ * Check if the completion request flag should be set in the last WQE.\n+ * Both pushed mbufs and WQEs are monitored and the completion request\n+ * flag is set if any of thresholds is reached.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param n_mbuf\n+ *   Number of mbuf not stored yet in elts array.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_request_completion(struct mlx5_txq_data *restrict txq,\n+\t\t\t   unsigned int n_mbuf,\n+\t\t\t   struct mlx5_txq_local *restrict loc,\n+\t\t\t   unsigned int olx __rte_unused)\n+{\n+\tuint16_t head = txq->elts_head + n_mbuf;\n+\n+\tif ((uint16_t)(head - txq->elts_comp) >= MLX5_TX_COMP_THRESH ||\n+\t    (uint16_t)(txq->wqe_ci - txq->wqe_comp) >= txq->wqe_thres) {\n+\t\tvolatile struct mlx5_wqe *last = loc->wqe_last;\n+\n+\t\ttxq->elts_comp = head;\n+\t\ttxq->wqe_comp = txq->wqe_ci;\n+\t\t/* Request unconditional completion on last WQE. */\n+\t\tlast->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<\n+\t\t\t\t\t    MLX5_COMP_MODE_OFFSET);\n+\t\t/* Save elts_head in unused \"immediate\" field of WQE. */\n+\t\tlast->cseg.misc = head;\n+\t\t/*\n+\t\t * A CQE slot must always be available. Count the\n+\t\t * issued CEQ \"always\" request instead of production\n+\t\t * index due to here can be CQE with errors and\n+\t\t * difference with ci may become inconsistent.\n+\t\t */\n+\t\tassert(txq->cqe_s > ++txq->cq_pi);\n+\t}\n+}\n+\n+/**\n  * DPDK callback to check the status of a tx descriptor.\n  *\n  * @param tx_queue\n@@ -1576,42 +1971,2490 @@ enum mlx5_txcmp_code {\n int\n mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)\n {\n-\t(void)tx_queue;\n-\t(void)offset;\n-\treturn RTE_ETH_TX_DESC_FULL;\n+\tstruct mlx5_txq_data *restrict txq = tx_queue;\n+\tuint16_t used;\n+\n+\tmlx5_tx_handle_completion(txq, 0);\n+\tused = txq->elts_head - txq->elts_tail;\n+\tif (offset < used)\n+\t\treturn RTE_ETH_TX_DESC_FULL;\n+\treturn RTE_ETH_TX_DESC_DONE;\n }\n \n /**\n- * DPDK Tx callback template. This is configured template\n- * used to generate routines optimized for specified offload setup.\n- * One of this generated functions is chosen at SQ configuration\n- * time.\n+ * Build the Control Segment with specified opcode:\n+ * - MLX5_OPCODE_SEND\n+ * - MLX5_OPCODE_ENHANCED_MPSW\n+ * - MLX5_OPCODE_TSO\n  *\n  * @param txq\n- *   Generic pointer to TX queue structure.\n- * @param[in] pkts\n- *   Packets to transmit.\n- * @param pkts_n\n- *   Number of packets in array.\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param wqe\n+ *   Pointer to WQE to fill with built Control Segment.\n+ * @param ds\n+ *   Supposed length of WQE in segments.\n+ * @param opcode\n+ *   SQ WQE opcode to put into Control Segment.\n  * @param olx\n- *   Configured offloads mask, presents the bits of MLX5_TXOFF_CONFIG_xxx\n- *   values. Should be static to take compile time static configuration\n- *   advantages.\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_cseg_init(struct mlx5_txq_data *restrict txq,\n+\t\t  struct mlx5_txq_local *restrict loc __rte_unused,\n+\t\t  struct mlx5_wqe *restrict wqe,\n+\t\t  unsigned int ds,\n+\t\t  unsigned int opcode,\n+\t\t  unsigned int olx __rte_unused)\n+{\n+\tstruct mlx5_wqe_cseg *restrict cs = &wqe->cseg;\n+\n+\tcs->opcode = rte_cpu_to_be_32((txq->wqe_ci << 8) | opcode);\n+\tcs->sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);\n+\tcs->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<\n+\t\t\t     MLX5_COMP_MODE_OFFSET);\n+\tcs->misc = RTE_BE32(0);\n+}\n+\n+/**\n+ * Build the Ethernet Segment without inlined data.\n+ * Supports Software Parser, Checksums and VLAN\n+ * insertion Tx offload features.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param wqe\n+ *   Pointer to WQE to fill with built Ethernet Segment.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_eseg_none(struct mlx5_txq_data *restrict txq __rte_unused,\n+\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t  struct mlx5_wqe *restrict wqe,\n+\t\t  unsigned int olx)\n+{\n+\tstruct mlx5_wqe_eseg *restrict es = &wqe->eseg;\n+\tuint32_t csum;\n+\n+\t/*\n+\t * Calculate and set check sum flags first, dword field\n+\t * in segment may be shared with Software Parser flags.\n+\t */\n+\tcsum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;\n+\tes->flags = rte_cpu_to_le_32(csum);\n+\t/*\n+\t * Calculate and set Software Parser offsets and flags.\n+\t * These flags a set for custom UDP and IP tunnel packets.\n+\t */\n+\tes->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);\n+\t/* Fill metadata field if needed. */\n+\tes->metadata = MLX5_TXOFF_CONFIG(METADATA) ?\n+\t\t       loc->mbuf->ol_flags & PKT_TX_METADATA ?\n+\t\t       loc->mbuf->tx_metadata : 0 : 0;\n+\t/* Engage VLAN tag insertion feature if requested. */\n+\tif (MLX5_TXOFF_CONFIG(VLAN) &&\n+\t    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)\n+\t\tes->inline_hdr = rte_cpu_to_be_32(MLX5_ETH_WQE_VLAN_INSERT |\n+\t\t\t\t\t\t  loc->mbuf->vlan_tci);\n+\telse\n+\t\tes->inline_hdr = RTE_BE32(0);\n+}\n+\n+/**\n+ * Build the Ethernet Segment with minimal inlined data\n+ * of MLX5_ESEG_MIN_INLINE_SIZE bytes length. This is\n+ * used to fill the gap in single WQEBB WQEs.\n+ * Supports Software Parser, Checksums and VLAN\n+ * insertion Tx offload features.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param wqe\n+ *   Pointer to WQE to fill with built Ethernet Segment.\n+ * @param vlan\n+ *   Length of VLAN tag insertion if any.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_eseg_dmin(struct mlx5_txq_data *restrict txq __rte_unused,\n+\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t  struct mlx5_wqe *restrict wqe,\n+\t\t  unsigned int vlan,\n+\t\t  unsigned int olx)\n+{\n+\tstruct mlx5_wqe_eseg *restrict es = &wqe->eseg;\n+\tuint32_t csum;\n+\tuint8_t *psrc, *pdst;\n+\n+\t/*\n+\t * Calculate and set check sum flags first, dword field\n+\t * in segment may be shared with Software Parser flags.\n+\t */\n+\tcsum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;\n+\tes->flags = rte_cpu_to_le_32(csum);\n+\t/*\n+\t * Calculate and set Software Parser offsets and flags.\n+\t * These flags a set for custom UDP and IP tunnel packets.\n+\t */\n+\tes->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);\n+\t/* Fill metadata field if needed. */\n+\tes->metadata = MLX5_TXOFF_CONFIG(METADATA) ?\n+\t\t       loc->mbuf->ol_flags & PKT_TX_METADATA ?\n+\t\t       loc->mbuf->tx_metadata : 0 : 0;\n+\tstatic_assert(MLX5_ESEG_MIN_INLINE_SIZE ==\n+\t\t\t\t(sizeof(uint16_t) +\n+\t\t\t\t sizeof(rte_v128u32_t)),\n+\t\t      \"invalid Ethernet Segment data size\");\n+\tstatic_assert(MLX5_ESEG_MIN_INLINE_SIZE ==\n+\t\t\t\t(sizeof(uint16_t) +\n+\t\t\t\t sizeof(struct rte_vlan_hdr) +\n+\t\t\t\t 2 * RTE_ETHER_ADDR_LEN),\n+\t\t      \"invalid Ethernet Segment data size\");\n+\tpsrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);\n+\tes->inline_hdr_sz = RTE_BE16(MLX5_ESEG_MIN_INLINE_SIZE);\n+\tes->inline_data = *(uint16_t *)psrc;\n+\tpsrc +=\tsizeof(uint16_t);\n+\tpdst = (uint8_t *)(es + 1);\n+\tif (MLX5_TXOFF_CONFIG(VLAN) && vlan) {\n+\t\t/* Implement VLAN tag insertion as part inline data. */\n+\t\tmemcpy(pdst, psrc, 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t));\n+\t\tpdst += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t);\n+\t\tpsrc +=\t2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t);\n+\t\t/* Insert VLAN ethertype + VLAN tag. */\n+\t\t*(uint32_t *)pdst = rte_cpu_to_be_32\n+\t\t\t\t\t((RTE_ETHER_TYPE_VLAN << 16) |\n+\t\t\t\t\t loc->mbuf->vlan_tci);\n+\t\tpdst += sizeof(struct rte_vlan_hdr);\n+\t\t/* Copy the rest two bytes from packet data. */\n+\t\t*(uint16_t *)pdst = *(uint16_t *)psrc;\n+\t} else {\n+\t\t/* Fill the gap in the title WQEBB with inline data. */\n+\t\trte_mov16(pdst, psrc);\n+\t}\n+}\n+\n+/**\n+ * Build the Ethernet Segment with entire packet\n+ * data inlining. Checks the boundary of WQEBB and\n+ * ring buffer wrapping, supports Software Parser,\n+ * Checksums and VLAN insertion Tx offload features.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param wqe\n+ *   Pointer to WQE to fill with built Ethernet Segment.\n+ * @param vlan\n+ *   Length of VLAN tag insertion if any.\n+ * @param inlen\n+ *   Length of data to inline (VLAN included, if any).\n+ * @param tso\n+ *   TSO flag, set mss field from the packet.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n  *\n  * @return\n- *   Number of packets successfully transmitted (<= pkts_n).\n+ *   Pointer to the next Data Segment (aligned and wrapped around).\n  */\n-static __rte_always_inline uint16_t\n-mlx5_tx_burst_tmpl(struct mlx5_txq_data *restrict txq,\n-\t\t   struct rte_mbuf **restrict pkts,\n-\t\t   uint16_t pkts_n,\n-\t\t   unsigned int olx)\n+static __rte_always_inline struct mlx5_wqe_dseg *\n+mlx5_tx_eseg_data(struct mlx5_txq_data *restrict txq,\n+\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t  struct mlx5_wqe *restrict wqe,\n+\t\t  unsigned int vlan,\n+\t\t  unsigned int inlen,\n+\t\t  unsigned int tso,\n+\t\t  unsigned int olx)\n {\n-\t(void)txq;\n-\t(void)pkts;\n-\t(void)pkts_n;\n-\t(void)olx;\n-\treturn 0;\n+\tstruct mlx5_wqe_eseg *restrict es = &wqe->eseg;\n+\tuint32_t csum;\n+\tuint8_t *psrc, *pdst;\n+\tunsigned int part;\n+\n+\t/*\n+\t * Calculate and set check sum flags first, dword field\n+\t * in segment may be shared with Software Parser flags.\n+\t */\n+\tcsum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;\n+\tif (tso) {\n+\t\tcsum <<= 24;\n+\t\tcsum |= loc->mbuf->tso_segsz;\n+\t\tes->flags = rte_cpu_to_be_32(csum);\n+\t} else {\n+\t\tes->flags = rte_cpu_to_le_32(csum);\n+\t}\n+\t/*\n+\t * Calculate and set Software Parser offsets and flags.\n+\t * These flags a set for custom UDP and IP tunnel packets.\n+\t */\n+\tes->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);\n+\t/* Fill metadata field if needed. */\n+\tes->metadata = MLX5_TXOFF_CONFIG(METADATA) ?\n+\t\t       loc->mbuf->ol_flags & PKT_TX_METADATA ?\n+\t\t       loc->mbuf->tx_metadata : 0 : 0;\n+\tstatic_assert(MLX5_ESEG_MIN_INLINE_SIZE ==\n+\t\t\t\t(sizeof(uint16_t) +\n+\t\t\t\t sizeof(rte_v128u32_t)),\n+\t\t      \"invalid Ethernet Segment data size\");\n+\tstatic_assert(MLX5_ESEG_MIN_INLINE_SIZE ==\n+\t\t\t\t(sizeof(uint16_t) +\n+\t\t\t\t sizeof(struct rte_vlan_hdr) +\n+\t\t\t\t 2 * RTE_ETHER_ADDR_LEN),\n+\t\t      \"invalid Ethernet Segment data size\");\n+\tpsrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);\n+\tes->inline_hdr_sz = RTE_BE16(inlen);\n+\tes->inline_data = *(uint16_t *)psrc;\n+\tpsrc +=\tsizeof(uint16_t);\n+\tpdst = (uint8_t *)(es + 1);\n+\tif (MLX5_TXOFF_CONFIG(VLAN) && vlan) {\n+\t\t/* Implement VLAN tag insertion as part inline data. */\n+\t\tmemcpy(pdst, psrc, 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t));\n+\t\tpdst += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t);\n+\t\tpsrc +=\t2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t);\n+\t\t/* Insert VLAN ethertype + VLAN tag. */\n+\t\t*(uint32_t *)pdst = rte_cpu_to_be_32\n+\t\t\t\t\t((RTE_ETHER_TYPE_VLAN << 16) |\n+\t\t\t\t\t loc->mbuf->vlan_tci);\n+\t\tpdst += sizeof(struct rte_vlan_hdr);\n+\t\t/* Copy the rest two bytes from packet data. */\n+\t\t*(uint16_t *)pdst = *(uint16_t *)psrc;\n+\t\tpsrc += sizeof(uint16_t);\n+\t} else {\n+\t\t/* Fill the gap in the title WQEBB with inline data. */\n+\t\trte_mov16(pdst, psrc);\n+\t\tpsrc += sizeof(rte_v128u32_t);\n+\t}\n+\tpdst = (uint8_t *)(es + 2);\n+\tassert(inlen >= MLX5_ESEG_MIN_INLINE_SIZE);\n+\tassert(pdst < (uint8_t *)txq->wqes_end);\n+\tinlen -= MLX5_ESEG_MIN_INLINE_SIZE;\n+\tif (!inlen) {\n+\t\tassert(pdst == RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE));\n+\t\treturn (struct mlx5_wqe_dseg *)pdst;\n+\t}\n+\t/*\n+\t * The WQEBB space availability is checked by caller.\n+\t * Here we should be aware of WQE ring buffer wraparound only.\n+\t */\n+\tpart = (uint8_t *)txq->wqes_end - pdst;\n+\tpart = RTE_MIN(part, inlen);\n+\tdo {\n+\t\trte_memcpy(pdst, psrc, part);\n+\t\tinlen -= part;\n+\t\tif (likely(!inlen)) {\n+\t\t\t/*\n+\t\t\t * If return value is not used by the caller\n+\t\t\t * the code below will be optimized out.\n+\t\t\t */\n+\t\t\tpdst += part;\n+\t\t\tpdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);\n+\t\t\tif (unlikely(pdst >= (uint8_t *)txq->wqes_end))\n+\t\t\t\tpdst = (uint8_t *)txq->wqes;\n+\t\t\treturn (struct mlx5_wqe_dseg *)pdst;\n+\t\t}\n+\t\tpdst = (uint8_t *)txq->wqes;\n+\t\tpsrc += part;\n+\t\tpart = inlen;\n+\t} while (true);\n+}\n+\n+/**\n+ * Copy data from chain of mbuf to the specified linear buffer.\n+ * Checksums and VLAN insertion Tx offload features. If data\n+ * from some mbuf copied completely this mbuf is freed. Local\n+ * structure is used to keep the byte stream state.\n+ *\n+ * @param pdst\n+ *   Pointer to the destination linear buffer.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param len\n+ *   Length of data to be copied.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_mseg_memcpy(uint8_t *pdst,\n+\t\t    struct mlx5_txq_local *restrict loc,\n+\t\t    unsigned int len,\n+\t\t    unsigned int olx __rte_unused)\n+{\n+\tstruct rte_mbuf *mbuf;\n+\tunsigned int part, dlen;\n+\tuint8_t *psrc;\n+\n+\tassert(len);\n+\tdo {\n+\t\t/* Allow zero length packets, must check first. */\n+\t\tdlen = rte_pktmbuf_data_len(loc->mbuf);\n+\t\tif (dlen <= loc->mbuf_off) {\n+\t\t\t/* Exhausted packet, just free. */\n+\t\t\tmbuf = loc->mbuf;\n+\t\t\tloc->mbuf = mbuf->next;\n+\t\t\trte_pktmbuf_free_seg(mbuf);\n+\t\t\tloc->mbuf_off = 0;\n+\t\t\tassert(loc->mbuf_nseg > 1);\n+\t\t\tassert(loc->mbuf);\n+\t\t\t--loc->mbuf_nseg;\n+\t\t\tcontinue;\n+\t\t}\n+\t\tdlen -= loc->mbuf_off;\n+\t\tpsrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);\n+\t\tpsrc += loc->mbuf_off;\n+\t\tpart = RTE_MIN(len, dlen);\n+\t\trte_memcpy(pdst, psrc, part);\n+\t\tloc->mbuf_off += part;\n+\t\tlen -= part;\n+\t\tif (!len) {\n+\t\t\tif (loc->mbuf_off >= rte_pktmbuf_data_len(loc->mbuf)) {\n+\t\t\t\tloc->mbuf_off = 0;\n+\t\t\t\t/* Exhausted packet, just free. */\n+\t\t\t\tmbuf = loc->mbuf;\n+\t\t\t\tloc->mbuf = mbuf->next;\n+\t\t\t\trte_pktmbuf_free_seg(mbuf);\n+\t\t\t\tloc->mbuf_off = 0;\n+\t\t\t\tassert(loc->mbuf_nseg >= 1);\n+\t\t\t\t--loc->mbuf_nseg;\n+\t\t\t}\n+\t\t\treturn;\n+\t\t}\n+\t\tpdst += part;\n+\t} while (true);\n+}\n+\n+/**\n+ * Build the Ethernet Segment with inlined data from\n+ * multi-segment packet. Checks the boundary of WQEBB\n+ * and ring buffer wrapping, supports Software Parser,\n+ * Checksums and VLAN insertion Tx offload features.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param wqe\n+ *   Pointer to WQE to fill with built Ethernet Segment.\n+ * @param vlan\n+ *   Length of VLAN tag insertion if any.\n+ * @param inlen\n+ *   Length of data to inline (VLAN included, if any).\n+ * @param tso\n+ *   TSO flag, set mss field from the packet.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   Pointer to the next Data Segment (aligned and\n+ *   possible NOT wrapped around - caller should do\n+ *   wrapping check on its own).\n+ */\n+static __rte_always_inline struct mlx5_wqe_dseg *\n+mlx5_tx_eseg_mdat(struct mlx5_txq_data *restrict txq,\n+\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t  struct mlx5_wqe *restrict wqe,\n+\t\t  unsigned int vlan,\n+\t\t  unsigned int inlen,\n+\t\t  unsigned int tso,\n+\t\t  unsigned int olx)\n+{\n+\tstruct mlx5_wqe_eseg *restrict es = &wqe->eseg;\n+\tuint32_t csum;\n+\tuint8_t *pdst;\n+\tunsigned int part;\n+\n+\t/*\n+\t * Calculate and set check sum flags first, uint32_t field\n+\t * in segment may be shared with Software Parser flags.\n+\t */\n+\tcsum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;\n+\tif (tso) {\n+\t\tcsum <<= 24;\n+\t\tcsum |= loc->mbuf->tso_segsz;\n+\t\tes->flags = rte_cpu_to_be_32(csum);\n+\t} else {\n+\t\tes->flags = rte_cpu_to_le_32(csum);\n+\t}\n+\t/*\n+\t * Calculate and set Software Parser offsets and flags.\n+\t * These flags a set for custom UDP and IP tunnel packets.\n+\t */\n+\tes->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);\n+\t/* Fill metadata field if needed. */\n+\tes->metadata = MLX5_TXOFF_CONFIG(METADATA) ?\n+\t\t       loc->mbuf->ol_flags & PKT_TX_METADATA ?\n+\t\t       loc->mbuf->tx_metadata : 0 : 0;\n+\tstatic_assert(MLX5_ESEG_MIN_INLINE_SIZE ==\n+\t\t\t\t(sizeof(uint16_t) +\n+\t\t\t\t sizeof(rte_v128u32_t)),\n+\t\t      \"invalid Ethernet Segment data size\");\n+\tstatic_assert(MLX5_ESEG_MIN_INLINE_SIZE ==\n+\t\t\t\t(sizeof(uint16_t) +\n+\t\t\t\t sizeof(struct rte_vlan_hdr) +\n+\t\t\t\t 2 * RTE_ETHER_ADDR_LEN),\n+\t\t      \"invalid Ethernet Segment data size\");\n+\tassert(inlen > MLX5_ESEG_MIN_INLINE_SIZE);\n+\tes->inline_hdr_sz = RTE_BE16(inlen);\n+\tpdst = (uint8_t *)&es->inline_data;\n+\tif (MLX5_TXOFF_CONFIG(VLAN) && vlan) {\n+\t\t/* Implement VLAN tag insertion as part inline data. */\n+\t\tmlx5_tx_mseg_memcpy(pdst, loc, 2 * RTE_ETHER_ADDR_LEN, olx);\n+\t\t*(uint32_t *)pdst = rte_cpu_to_be_32\n+\t\t\t\t\t((RTE_ETHER_TYPE_VLAN << 16) |\n+\t\t\t\t\t loc->mbuf->vlan_tci);\n+\t\tpdst += sizeof(struct rte_vlan_hdr);\n+\t\tinlen -= 2 * RTE_ETHER_ADDR_LEN + sizeof(struct rte_vlan_hdr);\n+\t}\n+\tassert(pdst < (uint8_t *)txq->wqes_end);\n+\t/*\n+\t * The WQEBB space availability is checked by caller.\n+\t * Here we should be aware of WQE ring buffer wraparound only.\n+\t */\n+\tpart = (uint8_t *)txq->wqes_end - pdst;\n+\tpart = RTE_MIN(part, inlen);\n+\tassert(part);\n+\tdo {\n+\t\tmlx5_tx_mseg_memcpy(pdst, loc, part, olx);\n+\t\tinlen -= part;\n+\t\tif (likely(!inlen)) {\n+\t\t\tpdst += part;\n+\t\t\tpdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);\n+\t\t\treturn (struct mlx5_wqe_dseg *)pdst;\n+\t\t}\n+\t\tpdst = (uint8_t *)txq->wqes;\n+\t\tpart = inlen;\n+\t} while (true);\n+}\n+\n+/**\n+ * Build the Data Segment of pointer type.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param dseg\n+ *   Pointer to WQE to fill with built Data Segment.\n+ * @param buf\n+ *   Data buffer to point.\n+ * @param len\n+ *   Data buffer length.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_dseg_ptr(struct mlx5_txq_data *restrict txq,\n+\t\t struct mlx5_txq_local *restrict loc,\n+\t\t struct mlx5_wqe_dseg *restrict dseg,\n+\t\t uint8_t *buf,\n+\t\t unsigned int len,\n+\t\t unsigned int olx __rte_unused)\n+\n+{\n+\tassert(len);\n+\tdseg->bcount = rte_cpu_to_be_32(len);\n+\tdseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf);\n+\tdseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf);\n+}\n+\n+/**\n+ * Build the Data Segment of pointer type or inline\n+ * if data length is less than buffer in minimal\n+ * Data Segment size.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param dseg\n+ *   Pointer to WQE to fill with built Data Segment.\n+ * @param buf\n+ *   Data buffer to point.\n+ * @param len\n+ *   Data buffer length.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_dseg_iptr(struct mlx5_txq_data *restrict txq,\n+\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t  struct mlx5_wqe_dseg *restrict dseg,\n+\t\t  uint8_t *buf,\n+\t\t  unsigned int len,\n+\t\t  unsigned int olx __rte_unused)\n+\n+{\n+\tuintptr_t dst, src;\n+\n+\tassert(len);\n+\tif (len > MLX5_DSEG_MIN_INLINE_SIZE) {\n+\t\tdseg->bcount = rte_cpu_to_be_32(len);\n+\t\tdseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf);\n+\t\tdseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf);\n+\n+\t\treturn;\n+\t}\n+\tdseg->bcount = rte_cpu_to_be_32(len | MLX5_ETH_WQE_DATA_INLINE);\n+\t/* Unrolled implementation of generic rte_memcpy. */\n+\tdst = (uintptr_t)&dseg->inline_data[0];\n+\tsrc = (uintptr_t)buf;\n+\tif (len & 0x08) {\n+\t\t*(uint64_t *)dst = *(uint64_t *)src;\n+\t\tdst += sizeof(uint64_t);\n+\t\tsrc += sizeof(uint64_t);\n+\t}\n+\tif (len & 0x04) {\n+\t\t*(uint32_t *)dst = *(uint32_t *)src;\n+\t\tdst += sizeof(uint32_t);\n+\t\tsrc += sizeof(uint32_t);\n+\t}\n+\tif (len & 0x02) {\n+\t\t*(uint16_t *)dst = *(uint16_t *)src;\n+\t\tdst += sizeof(uint16_t);\n+\t\tsrc += sizeof(uint16_t);\n+\t}\n+\tif (len & 0x01)\n+\t\t*(uint8_t *)dst = *(uint8_t *)src;\n+}\n+\n+/**\n+ * Build the Data Segment of inlined data from single\n+ * segment packet, no VLAN insertion.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param dseg\n+ *   Pointer to WQE to fill with built Data Segment.\n+ * @param buf\n+ *   Data buffer to point.\n+ * @param len\n+ *   Data buffer length.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   Pointer to the next Data Segment after inlined data.\n+ *   Ring buffer wraparound check is needed. We do not\n+ *   do it here because it may not be needed for the\n+ *   last packet in the eMPW session.\n+ */\n+static __rte_always_inline struct mlx5_wqe_dseg *\n+mlx5_tx_dseg_empw(struct mlx5_txq_data *restrict txq,\n+\t\t  struct mlx5_txq_local *restrict loc __rte_unused,\n+\t\t  struct mlx5_wqe_dseg *restrict dseg,\n+\t\t  uint8_t *buf,\n+\t\t  unsigned int len,\n+\t\t  unsigned int olx __rte_unused)\n+{\n+\tunsigned int part;\n+\tuint8_t *pdst;\n+\n+\tdseg->bcount = rte_cpu_to_be_32(len | MLX5_ETH_WQE_DATA_INLINE);\n+\tpdst = &dseg->inline_data[0];\n+\t/*\n+\t * The WQEBB space availability is checked by caller.\n+\t * Here we should be aware of WQE ring buffer wraparound only.\n+\t */\n+\tpart = (uint8_t *)txq->wqes_end - pdst;\n+\tpart = RTE_MIN(part, len);\n+\tdo {\n+\t\trte_memcpy(pdst, buf, part);\n+\t\tlen -= part;\n+\t\tif (likely(!len)) {\n+\t\t\tpdst += part;\n+\t\t\tpdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);\n+\t\t\t/* Note: no final wraparound check here. */\n+\t\t\treturn (struct mlx5_wqe_dseg *)pdst;\n+\t\t}\n+\t\tpdst = (uint8_t *)txq->wqes;\n+\t\tbuf += part;\n+\t\tpart = len;\n+\t} while (true);\n+}\n+\n+/**\n+ * Build the Data Segment of inlined data from single\n+ * segment packet with VLAN insertion.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param dseg\n+ *   Pointer to the dseg fill with built Data Segment.\n+ * @param buf\n+ *   Data buffer to point.\n+ * @param len\n+ *   Data buffer length.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   Pointer to the next Data Segment after inlined data.\n+ *   Ring buffer wraparound check is needed.\n+ */\n+static __rte_always_inline struct mlx5_wqe_dseg *\n+mlx5_tx_dseg_vlan(struct mlx5_txq_data *restrict txq,\n+\t\t  struct mlx5_txq_local *restrict loc __rte_unused,\n+\t\t  struct mlx5_wqe_dseg *restrict dseg,\n+\t\t  uint8_t *buf,\n+\t\t  unsigned int len,\n+\t\t  unsigned int olx __rte_unused)\n+\n+{\n+\tunsigned int part;\n+\tuint8_t *pdst;\n+\n+\tassert(len > MLX5_ESEG_MIN_INLINE_SIZE);\n+\tstatic_assert(MLX5_DSEG_MIN_INLINE_SIZE ==\n+\t\t\t\t (2 * RTE_ETHER_ADDR_LEN),\n+\t\t      \"invalid Data Segment data size\");\n+\tdseg->bcount = rte_cpu_to_be_32((len + sizeof(struct rte_vlan_hdr)) |\n+\t\t\t\t\tMLX5_ETH_WQE_DATA_INLINE);\n+\tpdst = &dseg->inline_data[0];\n+\tmemcpy(pdst, buf, MLX5_DSEG_MIN_INLINE_SIZE);\n+\tbuf += MLX5_DSEG_MIN_INLINE_SIZE;\n+\t/* Insert VLAN ethertype + VLAN tag. */\n+\t*(uint32_t *)pdst = rte_cpu_to_be_32((RTE_ETHER_TYPE_VLAN << 16) |\n+\t\t\t\t\t      loc->mbuf->vlan_tci);\n+\tpdst += sizeof(struct rte_vlan_hdr);\n+\tif (unlikely(pdst >= (uint8_t *)txq->wqes_end))\n+\t\tpdst = (uint8_t *)txq->wqes;\n+\t/*\n+\t * The WQEBB space availability is checked by caller.\n+\t * Here we should be aware of WQE ring buffer wraparound only.\n+\t */\n+\tpart = (uint8_t *)txq->wqes_end - pdst;\n+\tpart = RTE_MIN(part, len);\n+\tdo {\n+\t\trte_memcpy(pdst, buf, part);\n+\t\tlen -= part;\n+\t\tif (likely(!len)) {\n+\t\t\tpdst += part;\n+\t\t\tpdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);\n+\t\t\t/* Note: no final wraparound check here. */\n+\t\t\treturn (struct mlx5_wqe_dseg *)pdst;\n+\t\t}\n+\t\tpdst = (uint8_t *)txq->wqes;\n+\t\tbuf += part;\n+\t\tpart = len;\n+\t} while (true);\n+}\n+\n+/**\n+ * Build the Ethernet Segment with optionally inlined data with\n+ * VLAN insertion and following Data Segments (if any) from\n+ * multi-segment packet. Used by ordinary send and TSO.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param wqe\n+ *   Pointer to WQE to fill with built Ethernet/Data Segments.\n+ * @param vlan\n+ *   Length of VLAN header to insert, 0 means no VLAN insertion.\n+ * @param inlen\n+ *   Data length to inline. This is minimal amount of data bytes\n+ *   to be inlined. For TSO this parameter specifies exact value,\n+ *   for ordinary send routine can extend beyond specified value\n+ *   to provide better WQE space saving. This length includes\n+ *   VLAN header being inserted.\n+ * @param tso\n+ *   Zero means ordinary send, inlined data can be extended,\n+ *   otherwise this is TSO, inlined data length is fixed.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   Actual size of built WQE in segments.\n+ */\n+static __rte_always_inline unsigned int\n+mlx5_tx_mseg_build(struct mlx5_txq_data *restrict txq,\n+\t\t   struct mlx5_txq_local *restrict loc,\n+\t\t   struct mlx5_wqe *restrict wqe,\n+\t\t   unsigned int vlan,\n+\t\t   unsigned int inlen,\n+\t\t   unsigned int tso,\n+\t\t   unsigned int olx __rte_unused)\n+{\n+\tstruct mlx5_wqe_dseg *restrict dseg;\n+\tunsigned int ds;\n+\n+\tassert((rte_pktmbuf_pkt_len(loc->mbuf) + vlan) >= inlen);\n+\tloc->mbuf_nseg = NB_SEGS(loc->mbuf);\n+\tloc->mbuf_off = 0;\n+\n+\tdseg = mlx5_tx_eseg_mdat(txq, loc, wqe, vlan, inlen, tso, olx);\n+\tif (!loc->mbuf_nseg)\n+\t\tgoto dseg_done;\n+\t/*\n+\t * There are still some mbuf remaining, not inlined.\n+\t * The first mbuf may be partially inlined and we\n+\t * must process the possible non-zero data offset.\n+\t */\n+\tif (loc->mbuf_off) {\n+\t\tunsigned int dlen;\n+\t\tuint8_t *dptr;\n+\n+\t\t/*\n+\t\t * Exhausted packets must be dropped before.\n+\t\t * Non-zero offset means there are some data\n+\t\t * remained in the packet.\n+\t\t */\n+\t\tassert(loc->mbuf_off < rte_pktmbuf_data_len(loc->mbuf));\n+\t\tassert(rte_pktmbuf_data_len(loc->mbuf));\n+\t\tdptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + loc->mbuf_off;\n+\t\tdlen = rte_pktmbuf_data_len(loc->mbuf) - loc->mbuf_off;\n+\t\t/*\n+\t\t * Build the pointer/minimal data Data Segment.\n+\t\t * Do ring buffer wrapping check in advance.\n+\t\t */\n+\t\tif ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)\n+\t\t\tdseg = (struct mlx5_wqe_dseg *)txq->wqes;\n+\t\tmlx5_tx_dseg_iptr(txq, loc, dseg, dptr, dlen, olx);\n+\t\t/* Store the mbuf to be freed on completion. */\n+\t\tassert(loc->elts_free);\n+\t\ttxq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;\n+\t\t--loc->elts_free;\n+\t\t++dseg;\n+\t\tif (--loc->mbuf_nseg == 0)\n+\t\t\tgoto dseg_done;\n+\t\tloc->mbuf = loc->mbuf->next;\n+\t\tloc->mbuf_off = 0;\n+\t}\n+\tdo {\n+\t\tif (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) {\n+\t\t\tstruct rte_mbuf *mbuf;\n+\n+\t\t\t/* Zero length segment found, just skip. */\n+\t\t\tmbuf = loc->mbuf;\n+\t\t\tloc->mbuf = loc->mbuf->next;\n+\t\t\trte_pktmbuf_free_seg(mbuf);\n+\t\t\tif (--loc->mbuf_nseg == 0)\n+\t\t\t\tbreak;\n+\t\t} else {\n+\t\t\tif ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)\n+\t\t\t\tdseg = (struct mlx5_wqe_dseg *)txq->wqes;\n+\t\t\tmlx5_tx_dseg_iptr\n+\t\t\t\t(txq, loc, dseg,\n+\t\t\t\t rte_pktmbuf_mtod(loc->mbuf, uint8_t *),\n+\t\t\t\t rte_pktmbuf_data_len(loc->mbuf), olx);\n+\t\t\tassert(loc->elts_free);\n+\t\t\ttxq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;\n+\t\t\t--loc->elts_free;\n+\t\t\t++dseg;\n+\t\t\tif (--loc->mbuf_nseg == 0)\n+\t\t\t\tbreak;\n+\t\t\tloc->mbuf = loc->mbuf->next;\n+\t\t}\n+\t} while (true);\n+\n+dseg_done:\n+\t/* Calculate actual segments used from the dseg pointer. */\n+\tif ((uintptr_t)wqe < (uintptr_t)dseg)\n+\t\tds = ((uintptr_t)dseg - (uintptr_t)wqe) / MLX5_WSEG_SIZE;\n+\telse\n+\t\tds = (((uintptr_t)dseg - (uintptr_t)wqe) +\n+\t\t      txq->wqe_s * MLX5_WQE_SIZE) / MLX5_WSEG_SIZE;\n+\treturn ds;\n+}\n+\n+/**\n+ * Tx one packet function for multi-segment TSO. Supports all\n+ * types of Tx offloads, uses MLX5_OPCODE_TSO to build WQEs,\n+ * sends one packet per WQE.\n+ *\n+ * This routine is responsible for storing processed mbuf\n+ * into elts ring buffer and update elts_head.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   MLX5_TXCMP_CODE_EXIT - sending is done or impossible.\n+ *   MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.\n+ * Local context variables partially updated.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_packet_multi_tso(struct mlx5_txq_data *restrict txq,\n+\t\t\tstruct mlx5_txq_local *restrict loc,\n+\t\t\tunsigned int olx)\n+{\n+\tstruct mlx5_wqe *restrict wqe;\n+\tunsigned int ds, dlen, inlen, ntcp, vlan = 0;\n+\n+\t/*\n+\t * Calculate data length to be inlined to estimate\n+\t * the required space in WQE ring buffer.\n+\t */\n+\tdlen = rte_pktmbuf_pkt_len(loc->mbuf);\n+\tif (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)\n+\t\tvlan = sizeof(struct rte_vlan_hdr);\n+\tinlen = loc->mbuf->l2_len + vlan +\n+\t\tloc->mbuf->l3_len + loc->mbuf->l4_len;\n+\tif (unlikely((!inlen || !loc->mbuf->tso_segsz)))\n+\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\tif (loc->mbuf->ol_flags & PKT_TX_TUNNEL_MASK)\n+\t\tinlen += loc->mbuf->outer_l2_len + loc->mbuf->outer_l3_len;\n+\t/* Packet must contain all TSO headers. */\n+\tif (unlikely(inlen > MLX5_MAX_TSO_HEADER ||\n+\t\t     inlen <= MLX5_ESEG_MIN_INLINE_SIZE ||\n+\t\t     inlen > (dlen + vlan)))\n+\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t/*\n+\t * Check whether there are enough free WQEBBs:\n+\t * - Control Segment\n+\t * - Ethernet Segment\n+\t * - First Segment of inlined Ethernet data\n+\t * - ... data continued ...\n+\t * - Data Segments of pointer/min inline type\n+\t */\n+\tds = NB_SEGS(loc->mbuf) + 2 + (inlen -\n+\t\t\t\t       MLX5_ESEG_MIN_INLINE_SIZE +\n+\t\t\t\t       MLX5_WSEG_SIZE +\n+\t\t\t\t       MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;\n+\tif (unlikely(loc->wqe_free < ((ds + 3) / 4)))\n+\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t/* Check for maximal WQE size. */\n+\tif (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4)))\n+\t\treturn MLX5_TXCMP_CODE_ERROR;\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t/* Update sent data bytes/packets counters. */\n+\tntcp = (dlen - (inlen - vlan) + loc->mbuf->tso_segsz - 1) /\n+\t\tloc->mbuf->tso_segsz;\n+\t/*\n+\t * One will be added for mbuf itself\n+\t * at the end of the mlx5_tx_burst from\n+\t * loc->pkts_sent field.\n+\t */\n+\t--ntcp;\n+\ttxq->stats.opackets += ntcp;\n+\ttxq->stats.obytes += dlen + vlan + ntcp * inlen;\n+#endif\n+\twqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\tloc->wqe_last = wqe;\n+\tmlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_TSO, olx);\n+\tds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 1, olx);\n+\twqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);\n+\ttxq->wqe_ci += (ds + 3) / 4;\n+\tloc->wqe_free -= (ds + 3) / 4;\n+\treturn MLX5_TXCMP_CODE_MULTI;\n+}\n+\n+/**\n+ * Tx one packet function for multi-segment SEND. Supports all\n+ * types of Tx offloads, uses MLX5_OPCODE_SEND to build WQEs,\n+ * sends one packet per WQE, without any data inlining in\n+ * Ethernet Segment.\n+ *\n+ * This routine is responsible for storing processed mbuf\n+ * into elts ring buffer and update elts_head.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   MLX5_TXCMP_CODE_EXIT - sending is done or impossible.\n+ *   MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.\n+ * Local context variables partially updated.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_packet_multi_send(struct mlx5_txq_data *restrict txq,\n+\t\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t\t  unsigned int olx)\n+{\n+\tstruct mlx5_wqe_dseg *restrict dseg;\n+\tstruct mlx5_wqe *restrict wqe;\n+\tunsigned int ds, nseg;\n+\n+\tassert(NB_SEGS(loc->mbuf) > 1);\n+\t/*\n+\t * No inline at all, it means the CPU cycles saving\n+\t * is prioritized at configuration, we should not\n+\t * copy any packet data to WQE.\n+\t */\n+\tnseg = NB_SEGS(loc->mbuf);\n+\tds = 2 + nseg;\n+\tif (unlikely(loc->wqe_free < ((ds + 3) / 4)))\n+\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t/* Check for maximal WQE size. */\n+\tif (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4)))\n+\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t/*\n+\t * Some Tx offloads may cause an error if\n+\t * packet is not long enough, check against\n+\t * assumed minimal length.\n+\t */\n+\tif (rte_pktmbuf_pkt_len(loc->mbuf) <= MLX5_ESEG_MIN_INLINE_SIZE)\n+\t\treturn MLX5_TXCMP_CODE_ERROR;\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t/* Update sent data bytes counter. */\n+\ttxq->stats.obytes += rte_pktmbuf_pkt_len(loc->mbuf);\n+\tif (MLX5_TXOFF_CONFIG(VLAN) &&\n+\t    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)\n+\t\ttxq->stats.obytes += sizeof(struct rte_vlan_hdr);\n+#endif\n+\t/*\n+\t * SEND WQE, one WQEBB:\n+\t * - Control Segment, SEND opcode\n+\t * - Ethernet Segment, optional VLAN, no inline\n+\t * - Data Segments, pointer only type\n+\t */\n+\twqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\tloc->wqe_last = wqe;\n+\tmlx5_tx_cseg_init(txq, loc, wqe, ds, MLX5_OPCODE_SEND, olx);\n+\tmlx5_tx_eseg_none(txq, loc, wqe, olx);\n+\tdseg = &wqe->dseg[0];\n+\tdo {\n+\t\tif (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) {\n+\t\t\tstruct rte_mbuf *mbuf;\n+\n+\t\t\t/*\n+\t\t\t * Zero length segment found, have to\n+\t\t\t * correct total size of WQE in segments.\n+\t\t\t * It is supposed to be rare occasion, so\n+\t\t\t * in normal case (no zero length segments)\n+\t\t\t * we avoid extra writing to the Control\n+\t\t\t * Segment.\n+\t\t\t */\n+\t\t\t--ds;\n+\t\t\twqe->cseg.sq_ds -= RTE_BE32(1);\n+\t\t\tmbuf = loc->mbuf;\n+\t\t\tloc->mbuf = mbuf->next;\n+\t\t\trte_pktmbuf_free_seg(mbuf);\n+\t\t\tif (--nseg == 0)\n+\t\t\t\tbreak;\n+\t\t} else {\n+\t\t\tmlx5_tx_dseg_ptr\n+\t\t\t\t(txq, loc, dseg,\n+\t\t\t\t rte_pktmbuf_mtod(loc->mbuf, uint8_t *),\n+\t\t\t\t rte_pktmbuf_data_len(loc->mbuf), olx);\n+\t\t\ttxq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;\n+\t\t\t--loc->elts_free;\n+\t\t\tif (--nseg == 0)\n+\t\t\t\tbreak;\n+\t\t\t++dseg;\n+\t\t\tif ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)\n+\t\t\t\tdseg = (struct mlx5_wqe_dseg *)txq->wqes;\n+\t\t\tloc->mbuf = loc->mbuf->next;\n+\t\t}\n+\t} while (true);\n+\ttxq->wqe_ci += (ds + 3) / 4;\n+\tloc->wqe_free -= (ds + 3) / 4;\n+\treturn MLX5_TXCMP_CODE_MULTI;\n+}\n+\n+/**\n+ * Tx one packet function for multi-segment SEND. Supports all\n+ * types of Tx offloads, uses MLX5_OPCODE_SEND to build WQEs,\n+ * sends one packet per WQE, with data inlining in\n+ * Ethernet Segment and minimal Data Segments.\n+ *\n+ * This routine is responsible for storing processed mbuf\n+ * into elts ring buffer and update elts_head.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   MLX5_TXCMP_CODE_EXIT - sending is done or impossible.\n+ *   MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.\n+ * Local context variables partially updated.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_packet_multi_inline(struct mlx5_txq_data *restrict txq,\n+\t\t\t    struct mlx5_txq_local *restrict loc,\n+\t\t\t    unsigned int olx)\n+{\n+\tstruct mlx5_wqe *restrict wqe;\n+\tunsigned int ds, inlen, dlen, vlan = 0;\n+\n+\tassert(MLX5_TXOFF_CONFIG(INLINE));\n+\tassert(NB_SEGS(loc->mbuf) > 1);\n+\t/*\n+\t * First calculate data length to be inlined\n+\t * to estimate the required space for WQE.\n+\t */\n+\tdlen = rte_pktmbuf_pkt_len(loc->mbuf);\n+\tif (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)\n+\t\tvlan = sizeof(struct rte_vlan_hdr);\n+\tinlen = dlen + vlan;\n+\t/* Check against minimal length. */\n+\tif (inlen <= MLX5_ESEG_MIN_INLINE_SIZE)\n+\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\tassert(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE);\n+\tif (inlen > txq->inlen_send) {\n+\t\t/*\n+\t\t * Packet length exceeds the allowed inline\n+\t\t * data length, check whether the minimal\n+\t\t * inlining is required. eMPW check is here\n+\t\t * to statically exclude check - no minimal\n+\t\t * inlining required if eMPW is enabled.\n+\t\t */\n+\t\tif (MLX5_TXOFF_CONFIG(EMPW) || !txq->inlen_mode) {\n+\t\t\t/*\n+\t\t\t * VLAN insertion will be done inside by HW.\n+\t\t\t * It is not utmost effective - VLAN flag is\n+\t\t\t * checked twice, but we should proceed the\n+\t\t\t * inlining length correctly and take into\n+\t\t\t * account the VLAN header being inserted.\n+\t\t\t */\n+\t\t\tassert(!txq->inlen_mode);\n+\t\t\treturn mlx5_tx_packet_multi_send(txq, loc, olx);\n+\t\t}\n+\t\tassert(txq->inlen_mode >= MLX5_ESEG_MIN_INLINE_SIZE);\n+\t\tassert(txq->inlen_mode <= txq->inlen_send);\n+\t\tinlen = txq->inlen_mode;\n+\t}\n+\t/*\n+\t * Check whether there are enough free WQEBBs:\n+\t * - Control Segment\n+\t * - Ethernet Segment\n+\t * - First Segment of inlined Ethernet data\n+\t * - ... data continued ...\n+\t * - Data Segments of pointer/min inline type\n+\t *\n+\t * Estimate the number of Data Segments conservatively,\n+\t * supposing no any mbufs is being freed during inlining.\n+\t */\n+\tds = NB_SEGS(loc->mbuf) + 2 + (inlen -\n+\t\t\t\t       MLX5_ESEG_MIN_INLINE_SIZE +\n+\t\t\t\t       MLX5_WSEG_SIZE +\n+\t\t\t\t       MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;\n+\t/*\n+\t * We may have extra space in WQE to put inline\n+\t * data to fill the WQEBBs completely.\n+\t */\n+\tinlen += (4 - ds % 4) * MLX5_WSEG_SIZE;\n+\tinlen = RTE_MIN(inlen, dlen + vlan);\n+\tif (unlikely(loc->wqe_free < ((ds + 3) / 4)))\n+\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t/* Check for maximal WQE size. */\n+\tif (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4)))\n+\t\treturn MLX5_TXCMP_CODE_ERROR;\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t/* Update sent data bytes/packets counters. */\n+\ttxq->stats.obytes += dlen + vlan;\n+#endif\n+\twqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\tloc->wqe_last = wqe;\n+\tmlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_SEND, olx);\n+\tds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 0, olx);\n+\twqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);\n+\ttxq->wqe_ci += (ds + 3) / 4;\n+\tloc->wqe_free -= (ds + 3) / 4;\n+\treturn MLX5_TXCMP_CODE_MULTI;\n+}\n+\n+/**\n+ * Tx burst function for multi-segment packets. Supports all\n+ * types of Tx offloads, uses MLX5_OPCODE_SEND/TSO to build WQEs,\n+ * sends one packet per WQE. Function stops sending if it\n+ * encounters the single-segment packet.\n+ *\n+ * This routine is responsible for storing processed mbuf\n+ * into elts ring buffer and update elts_head.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param[in] pkts\n+ *   Packets to transmit.\n+ * @param pkts_n\n+ *   Number of packets in array.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   MLX5_TXCMP_CODE_EXIT - sending is done or impossible.\n+ *   MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.\n+ *   MLX5_TXCMP_CODE_SINGLE - single-segment packet encountered.\n+ *   MLX5_TXCMP_CODE_TSO - TSO single-segment packet encountered.\n+ * Local context variables updated.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_burst_mseg(struct mlx5_txq_data *restrict txq,\n+\t\t   struct rte_mbuf **restrict pkts,\n+\t\t   unsigned int pkts_n,\n+\t\t   struct mlx5_txq_local *restrict loc,\n+\t\t   unsigned int olx)\n+{\n+\tassert(loc->elts_free && loc->wqe_free);\n+\tassert(pkts_n > loc->pkts_sent);\n+\tpkts += loc->pkts_sent + 1;\n+\tpkts_n -= loc->pkts_sent;\n+\tfor (;;) {\n+\t\tenum mlx5_txcmp_code ret;\n+\n+\t\tassert(NB_SEGS(loc->mbuf) > 1);\n+\t\t/*\n+\t\t * Estimate the number of free elts quickly but\n+\t\t * conservatively. Some segment may be fully inlined\n+\t\t * and freed, ignore this here - precise estimation\n+\t\t * is costly.\n+\t\t */\n+\t\tif (loc->elts_free < NB_SEGS(loc->mbuf))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tif (MLX5_TXOFF_CONFIG(TSO) &&\n+\t\t    unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)) {\n+\t\t\t/* Proceed with multi-segment TSO. */\n+\t\t\tret = mlx5_tx_packet_multi_tso(txq, loc, olx);\n+\t\t} else if (MLX5_TXOFF_CONFIG(INLINE)) {\n+\t\t\t/* Proceed with multi-segment SEND with inlining. */\n+\t\t\tret = mlx5_tx_packet_multi_inline(txq, loc, olx);\n+\t\t} else {\n+\t\t\t/* Proceed with multi-segment SEND w/o inlining. */\n+\t\t\tret = mlx5_tx_packet_multi_send(txq, loc, olx);\n+\t\t}\n+\t\tif (ret == MLX5_TXCMP_CODE_EXIT)\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tif (ret == MLX5_TXCMP_CODE_ERROR)\n+\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t/* WQE is built, go to the next packet. */\n+\t\t++loc->pkts_sent;\n+\t\t--pkts_n;\n+\t\tif (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tloc->mbuf = *pkts++;\n+\t\tif (pkts_n > 1)\n+\t\t\trte_prefetch0(*pkts);\n+\t\tif (likely(NB_SEGS(loc->mbuf) > 1))\n+\t\t\tcontinue;\n+\t\t/* Here ends the series of multi-segment packets. */\n+\t\tif (MLX5_TXOFF_CONFIG(TSO) &&\n+\t\t    unlikely(!(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)))\n+\t\t\treturn MLX5_TXCMP_CODE_TSO;\n+\t\treturn MLX5_TXCMP_CODE_SINGLE;\n+\t}\n+\tassert(false);\n+}\n+\n+/**\n+ * Tx burst function for single-segment packets with TSO.\n+ * Supports all types of Tx offloads, except multi-packets.\n+ * Uses MLX5_OPCODE_TSO to build WQEs, sends one packet per WQE.\n+ * Function stops sending if it encounters the multi-segment\n+ * packet or packet without TSO requested.\n+ *\n+ * The routine is responsible for storing processed mbuf\n+ * into elts ring buffer and update elts_head if inline\n+ * offloads is requested due to possible early freeing\n+ * of the inlined mbufs (can not store pkts array in elts\n+ * as a batch).\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param[in] pkts\n+ *   Packets to transmit.\n+ * @param pkts_n\n+ *   Number of packets in array.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   MLX5_TXCMP_CODE_EXIT - sending is done or impossible.\n+ *   MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.\n+ *   MLX5_TXCMP_CODE_SINGLE - single-segment packet encountered.\n+ *   MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered.\n+ * Local context variables updated.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_burst_tso(struct mlx5_txq_data *restrict txq,\n+\t\t  struct rte_mbuf **restrict pkts,\n+\t\t  unsigned int pkts_n,\n+\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t  unsigned int olx)\n+{\n+\tassert(loc->elts_free && loc->wqe_free);\n+\tassert(pkts_n > loc->pkts_sent);\n+\tpkts += loc->pkts_sent + 1;\n+\tpkts_n -= loc->pkts_sent;\n+\tfor (;;) {\n+\t\tstruct mlx5_wqe_dseg *restrict dseg;\n+\t\tstruct mlx5_wqe *restrict wqe;\n+\t\tunsigned int ds, dlen, hlen, ntcp, vlan = 0;\n+\t\tuint8_t *dptr;\n+\n+\t\tassert(NB_SEGS(loc->mbuf) == 1);\n+\t\tdlen = rte_pktmbuf_data_len(loc->mbuf);\n+\t\tif (MLX5_TXOFF_CONFIG(VLAN) &&\n+\t\t    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {\n+\t\t\tvlan = sizeof(struct rte_vlan_hdr);\n+\t\t}\n+\t\t/*\n+\t\t * First calculate the WQE size to check\n+\t\t * whether we have enough space in ring buffer.\n+\t\t */\n+\t\thlen = loc->mbuf->l2_len + vlan +\n+\t\t       loc->mbuf->l3_len + loc->mbuf->l4_len;\n+\t\tif (unlikely((!hlen || !loc->mbuf->tso_segsz)))\n+\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\tif (loc->mbuf->ol_flags & PKT_TX_TUNNEL_MASK)\n+\t\t\thlen += loc->mbuf->outer_l2_len +\n+\t\t\t\tloc->mbuf->outer_l3_len;\n+\t\t/* Segment must contain all TSO headers. */\n+\t\tif (unlikely(hlen > MLX5_MAX_TSO_HEADER ||\n+\t\t\t     hlen <= MLX5_ESEG_MIN_INLINE_SIZE ||\n+\t\t\t     hlen > (dlen + vlan)))\n+\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t/*\n+\t\t * Check whether there are enough free WQEBBs:\n+\t\t * - Control Segment\n+\t\t * - Ethernet Segment\n+\t\t * - First Segment of inlined Ethernet data\n+\t\t * - ... data continued ...\n+\t\t * - Finishing Data Segment of pointer type\n+\t\t */\n+\t\tds = 4 + (hlen - MLX5_ESEG_MIN_INLINE_SIZE +\n+\t\t\t  MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;\n+\t\tif (loc->wqe_free < ((ds + 3) / 4))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t\t/* Update sent data bytes/packets counters. */\n+\t\tntcp = (dlen + vlan - hlen +\n+\t\t\tloc->mbuf->tso_segsz - 1) /\n+\t\t\tloc->mbuf->tso_segsz;\n+\t\t/*\n+\t\t * One will be added for mbuf itself at the end\n+\t\t * of the mlx5_tx_burst from loc->pkts_sent field.\n+\t\t */\n+\t\t--ntcp;\n+\t\ttxq->stats.opackets += ntcp;\n+\t\ttxq->stats.obytes += dlen + vlan + ntcp * hlen;\n+#endif\n+\t\t/*\n+\t\t * Build the TSO WQE:\n+\t\t * - Control Segment\n+\t\t * - Ethernet Segment with hlen bytes inlined\n+\t\t * - Data Segment of pointer type\n+\t\t */\n+\t\twqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\t\tloc->wqe_last = wqe;\n+\t\tmlx5_tx_cseg_init(txq, loc, wqe, ds,\n+\t\t\t\t  MLX5_OPCODE_TSO, olx);\n+\t\tdseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan, hlen, 1, olx);\n+\t\tdptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + hlen - vlan;\n+\t\tdlen -= hlen - vlan;\n+\t\tmlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);\n+\t\t/*\n+\t\t * WQE is built, update the loop parameters\n+\t\t * and go to the next packet.\n+\t\t */\n+\t\ttxq->wqe_ci += (ds + 3) / 4;\n+\t\tloc->wqe_free -= (ds + 3) / 4;\n+\t\tif (MLX5_TXOFF_CONFIG(INLINE))\n+\t\t\ttxq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;\n+\t\t--loc->elts_free;\n+\t\t++loc->pkts_sent;\n+\t\t--pkts_n;\n+\t\tif (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tloc->mbuf = *pkts++;\n+\t\tif (pkts_n > 1)\n+\t\t\trte_prefetch0(*pkts);\n+\t\tif (MLX5_TXOFF_CONFIG(MULTI) &&\n+\t\t    unlikely(NB_SEGS(loc->mbuf) > 1))\n+\t\t\treturn MLX5_TXCMP_CODE_MULTI;\n+\t\tif (unlikely(!(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)))\n+\t\t\treturn MLX5_TXCMP_CODE_SINGLE;\n+\t\t/* Continue with the next TSO packet. */\n+\t}\n+\tassert(false);\n+}\n+\n+/**\n+ * Analyze the packet and select the best method to send.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ * @param newp\n+ *   The predefined flag whether do complete check for\n+ *   multi-segment packets and TSO.\n+ *\n+ * @return\n+ *  MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered.\n+ *  MLX5_TXCMP_CODE_TSO - TSO required, use TSO/LSO.\n+ *  MLX5_TXCMP_CODE_SINGLE - single-segment packet, use SEND.\n+ *  MLX5_TXCMP_CODE_EMPW - single-segment packet, use MPW.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_able_to_empw(struct mlx5_txq_data *restrict txq,\n+\t\t     struct mlx5_txq_local *restrict loc,\n+\t\t     unsigned int olx,\n+\t\t     bool newp)\n+{\n+\t/* Check for multi-segment packet. */\n+\tif (newp &&\n+\t    MLX5_TXOFF_CONFIG(MULTI) &&\n+\t    unlikely(NB_SEGS(loc->mbuf) > 1))\n+\t\treturn MLX5_TXCMP_CODE_MULTI;\n+\t/* Check for TSO packet. */\n+\tif (newp &&\n+\t    MLX5_TXOFF_CONFIG(TSO) &&\n+\t    unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG))\n+\t\treturn MLX5_TXCMP_CODE_TSO;\n+\t/* Check if eMPW is enabled at all. */\n+\tif (!MLX5_TXOFF_CONFIG(EMPW))\n+\t\treturn MLX5_TXCMP_CODE_SINGLE;\n+\t/* Check if eMPW can be engaged. */\n+\tif (MLX5_TXOFF_CONFIG(VLAN) &&\n+\t    unlikely(loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) &&\n+\t\t(!MLX5_TXOFF_CONFIG(INLINE) ||\n+\t\t unlikely((rte_pktmbuf_data_len(loc->mbuf) +\n+\t\t\t   sizeof(struct rte_vlan_hdr)) > txq->inlen_empw))) {\n+\t\t/*\n+\t\t * eMPW does not support VLAN insertion offload,\n+\t\t * we have to inline the entire packet but\n+\t\t * packet is too long for inlining.\n+\t\t */\n+\t\treturn MLX5_TXCMP_CODE_SINGLE;\n+\t}\n+\treturn MLX5_TXCMP_CODE_EMPW;\n+}\n+\n+/**\n+ * Check the next packet attributes to match with the eMPW batch ones.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param es\n+ *   Pointer to Ethernet Segment of eMPW batch.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *  true - packet match with eMPW batch attributes.\n+ *  false - no match, eMPW should be restarted.\n+ */\n+static __rte_always_inline bool\n+mlx5_tx_match_empw(struct mlx5_txq_data *restrict txq __rte_unused,\n+\t\t   struct mlx5_wqe_eseg *restrict es,\n+\t\t   struct mlx5_txq_local *restrict loc,\n+\t\t   unsigned int olx)\n+{\n+\tuint8_t swp_flags = 0;\n+\n+\t/* Compare the checksum flags, if any. */\n+\tif (MLX5_TXOFF_CONFIG(CSUM) &&\n+\t    txq_ol_cksum_to_cs(loc->mbuf) != es->cs_flags)\n+\t\treturn false;\n+\t/* Compare the Software Parser offsets and flags. */\n+\tif (MLX5_TXOFF_CONFIG(SWP) &&\n+\t    (es->swp_offs != txq_mbuf_to_swp(loc, &swp_flags, olx) ||\n+\t     es->swp_flags != swp_flags))\n+\t\treturn false;\n+\t/* Fill metadata field if needed. */\n+\tif (MLX5_TXOFF_CONFIG(METADATA) &&\n+\t\tes->metadata != (loc->mbuf->ol_flags & PKT_TX_METADATA ?\n+\t\t\t\t loc->mbuf->tx_metadata : 0))\n+\t\treturn false;\n+\t/* There must be no VLAN packets in eMPW loop. */\n+\tif (MLX5_TXOFF_CONFIG(VLAN))\n+\t\tassert(!(loc->mbuf->ol_flags & PKT_TX_VLAN_PKT));\n+\treturn true;\n+}\n+\n+/*\n+ * Update send loop variables and WQE for eMPW loop\n+ * without data inlining. Number of Data Segments is\n+ * equal to the number of sent packets.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param ds\n+ *   Number of packets/Data Segments/Packets.\n+ * @param slen\n+ *   Accumulated statistics, bytes sent\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *  true - packet match with eMPW batch attributes.\n+ *  false - no match, eMPW should be restarted.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_sdone_empw(struct mlx5_txq_data *restrict txq,\n+\t\t   struct mlx5_txq_local *restrict loc,\n+\t\t   unsigned int ds,\n+\t\t   unsigned int slen,\n+\t\t   unsigned int olx __rte_unused)\n+{\n+\tassert(!MLX5_TXOFF_CONFIG(INLINE));\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t/* Update sent data bytes counter. */\n+\t txq->stats.obytes += slen;\n+#else\n+\t(void)slen;\n+#endif\n+\tloc->elts_free -= ds;\n+\tloc->pkts_sent += ds;\n+\tds += 2;\n+\tloc->wqe_last->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);\n+\ttxq->wqe_ci += (ds + 3) / 4;\n+\tloc->wqe_free -= (ds + 3) / 4;\n+}\n+\n+/*\n+ * Update send loop variables and WQE for eMPW loop\n+ * with data inlining. Gets the size of pushed descriptors\n+ * and data to the WQE.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param len\n+ *   Total size of descriptor/data in bytes.\n+ * @param slen\n+ *   Accumulated statistics, data bytes sent.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *  true - packet match with eMPW batch attributes.\n+ *  false - no match, eMPW should be restarted.\n+ */\n+static __rte_always_inline void\n+mlx5_tx_idone_empw(struct mlx5_txq_data *restrict txq,\n+\t\t   struct mlx5_txq_local *restrict loc,\n+\t\t   unsigned int len,\n+\t\t   unsigned int slen,\n+\t\t   unsigned int olx __rte_unused)\n+{\n+\tassert(MLX5_TXOFF_CONFIG(INLINE));\n+\tassert((len % MLX5_WSEG_SIZE) == 0);\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t/* Update sent data bytes counter. */\n+\t txq->stats.obytes += slen;\n+#else\n+\t(void)slen;\n+#endif\n+\tlen = len / MLX5_WSEG_SIZE + 2;\n+\tloc->wqe_last->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | len);\n+\ttxq->wqe_ci += (len + 3) / 4;\n+\tloc->wqe_free -= (len + 3) / 4;\n+}\n+\n+/**\n+ * Tx burst functions for single-segment packets without TSO\n+ * and with Multi-Packet Writing feature support. Supports\n+ * all types of Tx offloads, except multi-packets and TSO.\n+ * Uses MLX5_OPCODE_EMPW to build WQEs if possible and sends\n+ * as many packet per WQE as it can. If eMPW is not configured\n+ * or packet can not be sent with eMPW (VLAN insertion) the\n+ * ordinary SEND opcode is used and only one packet placed\n+ * in WQE.\n+ *\n+ * Function stops sending if it encounters the multi-segment\n+ * packet or packet with TSO requested.\n+ *\n+ * The routines are responsible for storing processed mbuf\n+ * into elts ring buffer and update elts_head if inlining\n+ * offload is requested. Otherwise the copying mbufs to elts\n+ * can be postponed and completed at the end of burst routine.\n+ *\n+ * @param txq\n+ *   Pointer to TX queue structure.\n+ * @param[in] pkts\n+ *   Packets to transmit.\n+ * @param pkts_n\n+ *   Number of packets in array.\n+ * @param loc\n+ *   Pointer to burst routine local context.\n+ * @param olx\n+ *   Configured Tx offloads mask. It is fully defined at\n+ *   compile time and may be used for optimization.\n+ *\n+ * @return\n+ *   MLX5_TXCMP_CODE_EXIT - sending is done or impossible.\n+ *   MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.\n+ *   MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered.\n+ *   MLX5_TXCMP_CODE_TSO - TSO packet encountered.\n+ * Local context variables updated.\n+ */\n+\n+/**\n+ * The routine sends packets with MLX5_OPCODE_EMPW\n+ * without inlining, this is dedicated optimized branch.\n+ * No VLAN insertion is supported.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_burst_empw_simple(struct mlx5_txq_data *restrict txq,\n+\t\t\t  struct rte_mbuf **restrict pkts,\n+\t\t\t  unsigned int pkts_n,\n+\t\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t\t  unsigned int olx)\n+{\n+\t/*\n+\t * Subroutine is the part of mlx5_tx_burst_single()\n+\t * and sends single-segment packet with eMPW opcode\n+\t * without data inlining.\n+\t */\n+\tassert(!MLX5_TXOFF_CONFIG(INLINE));\n+\tassert(MLX5_TXOFF_CONFIG(EMPW));\n+\tassert(loc->elts_free && loc->wqe_free);\n+\tassert(pkts_n > loc->pkts_sent);\n+\tstatic_assert(MLX5_EMPW_MIN_PACKETS >= 2, \"invalid min size\");\n+\tpkts += loc->pkts_sent + 1;\n+\tpkts_n -= loc->pkts_sent;\n+\tfor (;;) {\n+\t\tstruct mlx5_wqe_dseg *restrict dseg;\n+\t\tstruct mlx5_wqe_eseg *restrict eseg;\n+\t\tenum mlx5_txcmp_code ret;\n+\t\tunsigned int part, loop;\n+\t\tunsigned int slen = 0;\n+\n+next_empw:\n+\t\tpart = RTE_MIN(pkts_n, MLX5_EMPW_MAX_PACKETS);\n+\t\tif (unlikely(loc->elts_free < part)) {\n+\t\t\t/* We have no enough elts to save all mbufs. */\n+\t\t\tif (unlikely(loc->elts_free < MLX5_EMPW_MIN_PACKETS))\n+\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t/* But we still able to send at least minimal eMPW. */\n+\t\t\tpart = loc->elts_free;\n+\t\t}\n+\t\t/* Check whether we have enough WQEs */\n+\t\tif (unlikely(loc->wqe_free < ((2 + part + 3) / 4))) {\n+\t\t\tif (unlikely(loc->wqe_free <\n+\t\t\t\t((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4)))\n+\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\tpart = (loc->wqe_free * 4) - 2;\n+\t\t}\n+\t\tif (likely(part > 1))\n+\t\t\trte_prefetch0(*pkts);\n+\t\tloc->wqe_last = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\t\t/*\n+\t\t * Build eMPW title WQEBB:\n+\t\t * - Control Segment, eMPW opcode\n+\t\t * - Ethernet Segment, no inline\n+\t\t */\n+\t\tmlx5_tx_cseg_init(txq, loc, loc->wqe_last, part + 2,\n+\t\t\t\t  MLX5_OPCODE_ENHANCED_MPSW, olx);\n+\t\tmlx5_tx_eseg_none(txq, loc, loc->wqe_last,\n+\t\t\t\t  olx & ~MLX5_TXOFF_CONFIG_VLAN);\n+\t\teseg = &loc->wqe_last->eseg;\n+\t\tdseg = &loc->wqe_last->dseg[0];\n+\t\tloop = part;\n+\t\tfor (;;) {\n+\t\t\tuint32_t dlen = rte_pktmbuf_data_len(loc->mbuf);\n+\t\t\t/*\n+\t\t\t * Some Tx offloads may cause an error if\n+\t\t\t * packet is not long enough, check against\n+\t\t\t * assumed minimal length.\n+\t\t\t */\n+\t\t\tif (unlikely(dlen <= MLX5_ESEG_MIN_INLINE_SIZE)) {\n+\t\t\t\tpart -= loop;\n+\t\t\t\tif (unlikely(!part))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t\t\t/*\n+\t\t\t\t * We have some successfully built\n+\t\t\t\t * packet Data Segments to send.\n+\t\t\t\t */\n+\t\t\t\tmlx5_tx_sdone_empw(txq, loc, part, slen, olx);\n+\t\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t\t}\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t\t\t/* Update sent data bytes counter. */\n+\t\t\tslen += dlen;\n+#endif\n+\t\t\tmlx5_tx_dseg_ptr\n+\t\t\t\t(txq, loc, dseg,\n+\t\t\t\t rte_pktmbuf_mtod(loc->mbuf, uint8_t *),\n+\t\t\t\t dlen, olx);\n+\t\t\tif (unlikely(--loop == 0))\n+\t\t\t\tbreak;\n+\t\t\tloc->mbuf = *pkts++;\n+\t\t\tif (likely(loop > 1))\n+\t\t\t\trte_prefetch0(*pkts);\n+\t\t\tret = mlx5_tx_able_to_empw(txq, loc, olx, true);\n+\t\t\t/*\n+\t\t\t * Unroll the completion code to avoid\n+\t\t\t * returning variable value - it results in\n+\t\t\t * unoptimized sequent checking in caller.\n+\t\t\t */\n+\t\t\tif (ret == MLX5_TXCMP_CODE_MULTI) {\n+\t\t\t\tpart -= loop;\n+\t\t\t\tmlx5_tx_sdone_empw(txq, loc, part, slen, olx);\n+\t\t\t\tif (unlikely(!loc->elts_free ||\n+\t\t\t\t\t     !loc->wqe_free))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\treturn MLX5_TXCMP_CODE_MULTI;\n+\t\t\t}\n+\t\t\tif (ret == MLX5_TXCMP_CODE_TSO) {\n+\t\t\t\tpart -= loop;\n+\t\t\t\tmlx5_tx_sdone_empw(txq, loc, part, slen, olx);\n+\t\t\t\tif (unlikely(!loc->elts_free ||\n+\t\t\t\t\t     !loc->wqe_free))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\treturn MLX5_TXCMP_CODE_TSO;\n+\t\t\t}\n+\t\t\tif (ret == MLX5_TXCMP_CODE_SINGLE) {\n+\t\t\t\tpart -= loop;\n+\t\t\t\tmlx5_tx_sdone_empw(txq, loc, part, slen, olx);\n+\t\t\t\tif (unlikely(!loc->elts_free ||\n+\t\t\t\t\t     !loc->wqe_free))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\treturn MLX5_TXCMP_CODE_SINGLE;\n+\t\t\t}\n+\t\t\tif (ret != MLX5_TXCMP_CODE_EMPW) {\n+\t\t\t\tassert(false);\n+\t\t\t\tpart -= loop;\n+\t\t\t\tmlx5_tx_sdone_empw(txq, loc, part, slen, olx);\n+\t\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t\t}\n+\t\t\t/*\n+\t\t\t * Check whether packet parameters coincide\n+\t\t\t * within assumed eMPW batch:\n+\t\t\t * - check sum settings\n+\t\t\t * - metadata value\n+\t\t\t * - software parser settings\n+\t\t\t */\n+\t\t\tif (!mlx5_tx_match_empw(txq, eseg, loc, olx)) {\n+\t\t\t\tassert(loop);\n+\t\t\t\tpart -= loop;\n+\t\t\t\tmlx5_tx_sdone_empw(txq, loc, part, slen, olx);\n+\t\t\t\tif (unlikely(!loc->elts_free ||\n+\t\t\t\t\t     !loc->wqe_free))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\tgoto next_empw;\n+\t\t\t}\n+\t\t\t/* Packet attributes match, continue the same eMPW. */\n+\t\t\t++dseg;\n+\t\t\tif ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)\n+\t\t\t\tdseg = (struct mlx5_wqe_dseg *)txq->wqes;\n+\t\t}\n+\t\t/* eMPW is built successfully, update loop parameters. */\n+\t\tassert(!loop);\n+\t\tassert(pkts_n >= part);\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t\t/* Update sent data bytes counter. */\n+\t\ttxq->stats.obytes += slen;\n+#endif\n+\t\tloc->elts_free -= part;\n+\t\tloc->pkts_sent += part;\n+\t\ttxq->wqe_ci += (2 + part + 3) / 4;\n+\t\tloc->wqe_free -= (2 + part + 3) / 4;\n+\t\tpkts_n -= part;\n+\t\tif (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tloc->mbuf = *pkts++;\n+\t\tret = mlx5_tx_able_to_empw(txq, loc, olx, true);\n+\t\tif (unlikely(ret != MLX5_TXCMP_CODE_EMPW))\n+\t\t\treturn ret;\n+\t\t/* Continue sending eMPW batches. */\n+\t}\n+\tassert(false);\n+}\n+\n+/**\n+ * The routine sends packets with MLX5_OPCODE_EMPW\n+ * with inlining, optionally supports VLAN insertion.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_burst_empw_inline(struct mlx5_txq_data *restrict txq,\n+\t\t\t  struct rte_mbuf **restrict pkts,\n+\t\t\t  unsigned int pkts_n,\n+\t\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t\t  unsigned int olx)\n+{\n+\t/*\n+\t * Subroutine is the part of mlx5_tx_burst_single()\n+\t * and sends single-segment packet with eMPW opcode\n+\t * with data inlining.\n+\t */\n+\tassert(MLX5_TXOFF_CONFIG(INLINE));\n+\tassert(MLX5_TXOFF_CONFIG(EMPW));\n+\tassert(loc->elts_free && loc->wqe_free);\n+\tassert(pkts_n > loc->pkts_sent);\n+\tstatic_assert(MLX5_EMPW_MIN_PACKETS >= 2, \"invalid min size\");\n+\tpkts += loc->pkts_sent + 1;\n+\tpkts_n -= loc->pkts_sent;\n+\tfor (;;) {\n+\t\tstruct mlx5_wqe_dseg *restrict dseg;\n+\t\tstruct mlx5_wqe_eseg *restrict eseg;\n+\t\tenum mlx5_txcmp_code ret;\n+\t\tunsigned int room, part;\n+\t\tunsigned int slen = 0;\n+\n+next_empw:\n+\t\t/* Check whether we have minimal amount WQEs */\n+\t\tif (unlikely(loc->wqe_free <\n+\t\t\t    ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4)))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tif (likely(pkts_n > 1))\n+\t\t\trte_prefetch0(*pkts);\n+\t\tloc->wqe_last = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\t\t/*\n+\t\t * Build eMPW title WQEBB:\n+\t\t * - Control Segment, eMPW opcode, zero DS\n+\t\t * - Ethernet Segment, no inline\n+\t\t */\n+\t\tmlx5_tx_cseg_init(txq, loc, loc->wqe_last, 0,\n+\t\t\t\t  MLX5_OPCODE_ENHANCED_MPSW, olx);\n+\t\tmlx5_tx_eseg_none(txq, loc, loc->wqe_last,\n+\t\t\t\t  olx & ~MLX5_TXOFF_CONFIG_VLAN);\n+\t\teseg = &loc->wqe_last->eseg;\n+\t\tdseg = &loc->wqe_last->dseg[0];\n+\t\troom = RTE_MIN(MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE,\n+\t\t\t       loc->wqe_free) * MLX5_WQE_SIZE -\n+\t\t\t\t\tMLX5_WQE_CSEG_SIZE -\n+\t\t\t\t\tMLX5_WQE_ESEG_SIZE;\n+\t\t/* Build WQE till we have space, packets and resources. */\n+\t\tpart = room;\n+\t\tfor (;;) {\n+\t\t\tuint32_t dlen = rte_pktmbuf_data_len(loc->mbuf);\n+\t\t\tuint8_t *dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);\n+\t\t\tunsigned int tlen;\n+\n+\t\t\tassert(room >= MLX5_WQE_DSEG_SIZE);\n+\t\t\tassert((room % MLX5_WQE_DSEG_SIZE) == 0);\n+\t\t\tassert((uintptr_t)dseg < (uintptr_t)txq->wqes_end);\n+\t\t\t/*\n+\t\t\t * Some Tx offloads may cause an error if\n+\t\t\t * packet is not long enough, check against\n+\t\t\t * assumed minimal length.\n+\t\t\t */\n+\t\t\tif (unlikely(dlen <= MLX5_ESEG_MIN_INLINE_SIZE)) {\n+\t\t\t\tpart -= room;\n+\t\t\t\tif (unlikely(!part))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t\t\t/*\n+\t\t\t\t * We have some successfully built\n+\t\t\t\t * packet Data Segments to send.\n+\t\t\t\t */\n+\t\t\t\tmlx5_tx_idone_empw(txq, loc, part, slen, olx);\n+\t\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t\t}\n+\t\t\t/* Inline or not inline - that's the Question. */\n+\t\t\tif (dlen > txq->inlen_empw)\n+\t\t\t\tgoto pointer_empw;\n+\t\t\t/* Inline entire packet, optional VLAN insertion. */\n+\t\t\ttlen = sizeof(dseg->bcount) + dlen;\n+\t\t\tif (MLX5_TXOFF_CONFIG(VLAN) &&\n+\t\t\t    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {\n+\t\t\t\t/*\n+\t\t\t\t * The packet length must be checked in\n+\t\t\t\t * mlx5_tx_able_to_empw() and packet\n+\t\t\t\t * fits into inline length guaranteed.\n+\t\t\t\t */\n+\t\t\t\tassert((dlen + sizeof(struct rte_vlan_hdr)) <=\n+\t\t\t\t\ttxq->inlen_empw);\n+\t\t\t\ttlen += sizeof(struct rte_vlan_hdr);\n+\t\t\t\tif (room < tlen)\n+\t\t\t\t\tbreak;\n+\t\t\t\tdseg = mlx5_tx_dseg_vlan(txq, loc, dseg,\n+\t\t\t\t\t\t\t dptr, dlen, olx);\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t\t\t\t/* Update sent data bytes counter. */\n+\t\t\t\tslen +=\tsizeof(struct rte_vlan_hdr);\n+#endif\n+\t\t\t} else {\n+\t\t\t\tif (room < tlen)\n+\t\t\t\t\tbreak;\n+\t\t\t\tdseg = mlx5_tx_dseg_empw(txq, loc, dseg,\n+\t\t\t\t\t\t\t dptr, dlen, olx);\n+\t\t\t}\n+\t\t\ttlen = RTE_ALIGN(tlen, MLX5_WSEG_SIZE);\n+\t\t\tassert(room >= tlen);\n+\t\t\troom -= tlen;\n+\t\t\t/*\n+\t\t\t * Packet data are completely inlined,\n+\t\t\t * free the packet immediately.\n+\t\t\t */\n+\t\t\trte_pktmbuf_free_seg(loc->mbuf);\n+\t\t\tgoto next_mbuf;\n+pointer_empw:\n+\t\t\t/*\n+\t\t\t * Not inlinable VLAN packets are\n+\t\t\t * proceeded outside of this routine.\n+\t\t\t */\n+\t\t\tassert(room >= MLX5_WQE_DSEG_SIZE);\n+\t\t\tif (MLX5_TXOFF_CONFIG(VLAN))\n+\t\t\t\tassert(!(loc->mbuf->ol_flags &\n+\t\t\t\t\t PKT_TX_VLAN_PKT));\n+\t\t\tmlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);\n+\t\t\t/* We have to store mbuf in elts.*/\n+\t\t\ttxq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;\n+\t\t\troom -= MLX5_WQE_DSEG_SIZE;\n+\t\t\t/* Ring buffer wraparound is checked at the loop end.*/\n+\t\t\t++dseg;\n+next_mbuf:\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t\t\t/* Update sent data bytes counter. */\n+\t\t\tslen += dlen;\n+#endif\n+\t\t\tloc->pkts_sent++;\n+\t\t\tloc->elts_free--;\n+\t\t\tpkts_n--;\n+\t\t\tif (unlikely(!pkts_n || !loc->elts_free)) {\n+\t\t\t\t/*\n+\t\t\t\t * We have no resources/packets to\n+\t\t\t\t * continue build descriptors.\n+\t\t\t\t */\n+\t\t\t\tpart -= room;\n+\t\t\t\tmlx5_tx_idone_empw(txq, loc, part, slen, olx);\n+\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t}\n+\t\t\t/* Check if we have minimal room left. */\n+\t\t\tif (room < MLX5_WQE_DSEG_SIZE) {\n+\t\t\t\tpart -= room;\n+\t\t\t\tmlx5_tx_idone_empw(txq, loc, part, slen, olx);\n+\t\t\t\tgoto next_empw;\n+\t\t\t}\n+\t\t\tloc->mbuf = *pkts++;\n+\t\t\tif (likely(pkts_n > 1))\n+\t\t\t\trte_prefetch0(*pkts);\n+\t\t\tret = mlx5_tx_able_to_empw(txq, loc, olx, true);\n+\t\t\t/*\n+\t\t\t * Unroll the completion code to avoid\n+\t\t\t * returning variable value - it results in\n+\t\t\t * unoptimized sequent checking in caller.\n+\t\t\t */\n+\t\t\tif (ret == MLX5_TXCMP_CODE_MULTI) {\n+\t\t\t\tpart -= room;\n+\t\t\t\tmlx5_tx_idone_empw(txq, loc, part, slen, olx);\n+\t\t\t\tif (unlikely(!loc->elts_free ||\n+\t\t\t\t\t     !loc->wqe_free))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\treturn MLX5_TXCMP_CODE_MULTI;\n+\t\t\t}\n+\t\t\tif (ret == MLX5_TXCMP_CODE_TSO) {\n+\t\t\t\tpart -= room;\n+\t\t\t\tmlx5_tx_idone_empw(txq, loc, part, slen, olx);\n+\t\t\t\tif (unlikely(!loc->elts_free ||\n+\t\t\t\t\t     !loc->wqe_free))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\treturn MLX5_TXCMP_CODE_TSO;\n+\t\t\t}\n+\t\t\tif (ret == MLX5_TXCMP_CODE_SINGLE) {\n+\t\t\t\tpart -= room;\n+\t\t\t\tmlx5_tx_idone_empw(txq, loc, part, slen, olx);\n+\t\t\t\tif (unlikely(!loc->elts_free ||\n+\t\t\t\t\t     !loc->wqe_free))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\treturn MLX5_TXCMP_CODE_SINGLE;\n+\t\t\t}\n+\t\t\tif (ret != MLX5_TXCMP_CODE_EMPW) {\n+\t\t\t\tassert(false);\n+\t\t\t\tpart -= room;\n+\t\t\t\tmlx5_tx_idone_empw(txq, loc, part, slen, olx);\n+\t\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t\t}\n+\t\t\t/*\n+\t\t\t * Check whether packet parameters coincide\n+\t\t\t * within assumed eMPW batch:\n+\t\t\t * - check sum settings\n+\t\t\t * - metadata value\n+\t\t\t * - software parser settings\n+\t\t\t */\n+\t\t\tif (!mlx5_tx_match_empw(txq, eseg, loc, olx))\n+\t\t\t\tbreak;\n+\t\t\t/* Packet attributes match, continue the same eMPW. */\n+\t\t\tif ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)\n+\t\t\t\tdseg = (struct mlx5_wqe_dseg *)txq->wqes;\n+\t\t}\n+\t\t/*\n+\t\t * We get here to close an existing eMPW\n+\t\t * session and start the new one.\n+\t\t */\n+\t\tassert(pkts_n);\n+\t\tpart -= room;\n+\t\tif (unlikely(!part))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tmlx5_tx_idone_empw(txq, loc, part, slen, olx);\n+\t\tif (unlikely(!loc->elts_free ||\n+\t\t\t     !loc->wqe_free))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tgoto next_empw;\n+\t}\n+\tassert(false);\n+}\n+\n+/**\n+ * The routine sends packets with ordinary MLX5_OPCODE_SEND.\n+ * Data inlining and VLAN insertion are supported.\n+ */\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_burst_single_send(struct mlx5_txq_data *restrict txq,\n+\t\t\t  struct rte_mbuf **restrict pkts,\n+\t\t\t  unsigned int pkts_n,\n+\t\t\t  struct mlx5_txq_local *restrict loc,\n+\t\t\t  unsigned int olx)\n+{\n+\t/*\n+\t * Subroutine is the part of mlx5_tx_burst_single()\n+\t * and sends single-segment packet with SEND opcode.\n+\t */\n+\tassert(loc->elts_free && loc->wqe_free);\n+\tassert(pkts_n > loc->pkts_sent);\n+\tpkts += loc->pkts_sent + 1;\n+\tpkts_n -= loc->pkts_sent;\n+\tfor (;;) {\n+\t\tstruct mlx5_wqe *restrict wqe;\n+\t\tenum mlx5_txcmp_code ret;\n+\n+\t\tassert(NB_SEGS(loc->mbuf) == 1);\n+\t\tif (MLX5_TXOFF_CONFIG(INLINE)) {\n+\t\t\tunsigned int inlen, vlan = 0;\n+\n+\t\t\tinlen = rte_pktmbuf_data_len(loc->mbuf);\n+\t\t\tif (MLX5_TXOFF_CONFIG(VLAN) &&\n+\t\t\t    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {\n+\t\t\t\tvlan = sizeof(struct rte_vlan_hdr);\n+\t\t\t\tinlen += vlan;\n+\t\t\t\tstatic_assert((sizeof(struct rte_vlan_hdr) +\n+\t\t\t\t\t       sizeof(struct rte_ether_hdr)) ==\n+\t\t\t\t\t       MLX5_ESEG_MIN_INLINE_SIZE,\n+\t\t\t\t\t       \"invalid min inline data size\");\n+\t\t\t}\n+\t\t\t/*\n+\t\t\t * If inlining is enabled at configuration time\n+\t\t\t * the limit must be not less than minimal size.\n+\t\t\t * Otherwise we would do extra check for data\n+\t\t\t * size to avoid crashes due to length overflow.\n+\t\t\t */\n+\t\t\tassert(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE);\n+\t\t\tif (inlen <= txq->inlen_send) {\n+\t\t\t\tunsigned int seg_n, wqe_n;\n+\n+\t\t\t\trte_prefetch0(rte_pktmbuf_mtod\n+\t\t\t\t\t\t(loc->mbuf, uint8_t *));\n+\t\t\t\t/* Check against minimal length. */\n+\t\t\t\tif (inlen <= MLX5_ESEG_MIN_INLINE_SIZE)\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t\t\t/*\n+\t\t\t\t * Completely inlined packet data WQE:\n+\t\t\t\t * - Control Segment, SEND opcode\n+\t\t\t\t * - Ethernet Segment, no VLAN insertion\n+\t\t\t\t * - Data inlined, VLAN optionally inserted\n+\t\t\t\t * - Alignment to MLX5_WSEG_SIZE\n+\t\t\t\t * Have to estimate amount of WQEBBs\n+\t\t\t\t */\n+\t\t\t\tseg_n = (inlen + 3 * MLX5_WSEG_SIZE -\n+\t\t\t\t\t MLX5_ESEG_MIN_INLINE_SIZE +\n+\t\t\t\t\t MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;\n+\t\t\t\t/* Check if there are enough WQEBBs. */\n+\t\t\t\twqe_n = (seg_n + 3) / 4;\n+\t\t\t\tif (wqe_n > loc->wqe_free)\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\twqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\t\t\t\tloc->wqe_last = wqe;\n+\t\t\t\tmlx5_tx_cseg_init(txq, loc, wqe, seg_n,\n+\t\t\t\t\t\t  MLX5_OPCODE_SEND, olx);\n+\t\t\t\tmlx5_tx_eseg_data(txq, loc, wqe,\n+\t\t\t\t\t\t  vlan, inlen, 0, olx);\n+\t\t\t\ttxq->wqe_ci += wqe_n;\n+\t\t\t\tloc->wqe_free -= wqe_n;\n+\t\t\t\t/*\n+\t\t\t\t * Packet data are completely inlined,\n+\t\t\t\t * free the packet immediately.\n+\t\t\t\t */\n+\t\t\t\trte_pktmbuf_free_seg(loc->mbuf);\n+\t\t\t} else if (!MLX5_TXOFF_CONFIG(EMPW) &&\n+\t\t\t\t   txq->inlen_mode) {\n+\t\t\t\t/*\n+\t\t\t\t * If minimal inlining is requested the eMPW\n+\t\t\t\t * feature should be disabled due to data is\n+\t\t\t\t * inlined into Ethernet Segment, which can\n+\t\t\t\t * not contain inlined data for eMPW due to\n+\t\t\t\t * segment shared for all packets.\n+\t\t\t\t */\n+\t\t\t\tstruct mlx5_wqe_dseg *restrict dseg;\n+\t\t\t\tunsigned int ds;\n+\t\t\t\tuint8_t *dptr;\n+\n+\t\t\t\t/*\n+\t\t\t\t * The inline-mode settings require\n+\t\t\t\t * to inline the specified amount of\n+\t\t\t\t * data bytes to the Ethernet Segment.\n+\t\t\t\t * We should check the free space in\n+\t\t\t\t * WQE ring buffer to inline partially.\n+\t\t\t\t */\n+\t\t\t\tassert(txq->inlen_send >= txq->inlen_mode);\n+\t\t\t\tassert(inlen > txq->inlen_mode);\n+\t\t\t\tassert(txq->inlen_mode >=\n+\t\t\t\t\t\tMLX5_ESEG_MIN_INLINE_SIZE);\n+\t\t\t\t/*\n+\t\t\t\t * Check whether there are enough free WQEBBs:\n+\t\t\t\t * - Control Segment\n+\t\t\t\t * - Ethernet Segment\n+\t\t\t\t * - First Segment of inlined Ethernet data\n+\t\t\t\t * - ... data continued ...\n+\t\t\t\t * - Finishing Data Segment of pointer type\n+\t\t\t\t */\n+\t\t\t\tds = (MLX5_WQE_CSEG_SIZE +\n+\t\t\t\t      MLX5_WQE_ESEG_SIZE +\n+\t\t\t\t      MLX5_WQE_DSEG_SIZE +\n+\t\t\t\t      txq->inlen_mode -\n+\t\t\t\t      MLX5_ESEG_MIN_INLINE_SIZE +\n+\t\t\t\t      MLX5_WQE_DSEG_SIZE +\n+\t\t\t\t      MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;\n+\t\t\t\tif (loc->wqe_free < ((ds + 3) / 4))\n+\t\t\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\t\t\t/*\n+\t\t\t\t * Build the ordinary SEND WQE:\n+\t\t\t\t * - Control Segment\n+\t\t\t\t * - Ethernet Segment, inline inlen_mode bytes\n+\t\t\t\t * - Data Segment of pointer type\n+\t\t\t\t */\n+\t\t\t\twqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\t\t\t\tloc->wqe_last = wqe;\n+\t\t\t\tmlx5_tx_cseg_init(txq, loc, wqe, ds,\n+\t\t\t\t\t\t  MLX5_OPCODE_SEND, olx);\n+\t\t\t\tdseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan,\n+\t\t\t\t\t\t\t txq->inlen_mode,\n+\t\t\t\t\t\t\t 0, olx);\n+\t\t\t\tdptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) +\n+\t\t\t\t       txq->inlen_mode - vlan;\n+\t\t\t\tinlen -= txq->inlen_mode;\n+\t\t\t\tmlx5_tx_dseg_ptr(txq, loc, dseg,\n+\t\t\t\t\t\t dptr, inlen, olx);\n+\t\t\t\t/*\n+\t\t\t\t * WQE is built, update the loop parameters\n+\t\t\t\t * and got to the next packet.\n+\t\t\t\t */\n+\t\t\t\ttxq->wqe_ci += (ds + 3) / 4;\n+\t\t\t\tloc->wqe_free -= (ds + 3) / 4;\n+\t\t\t\t/* We have to store mbuf in elts.*/\n+\t\t\t\tassert(MLX5_TXOFF_CONFIG(INLINE));\n+\t\t\t\ttxq->elts[txq->elts_head++ & txq->elts_m] =\n+\t\t\t\t\t\tloc->mbuf;\n+\t\t\t\t--loc->elts_free;\n+\t\t\t} else {\n+\t\t\t\tuint8_t *dptr;\n+\t\t\t\tunsigned int dlen;\n+\n+\t\t\t\t/*\n+\t\t\t\t * Partially inlined packet data WQE, we have\n+\t\t\t\t * some space in title WQEBB, we can fill it\n+\t\t\t\t * with some packet data. It takes one WQEBB,\n+\t\t\t\t * it is available, no extra space check:\n+\t\t\t\t * - Control Segment, SEND opcode\n+\t\t\t\t * - Ethernet Segment, no VLAN insertion\n+\t\t\t\t * - MLX5_ESEG_MIN_INLINE_SIZE bytes of Data\n+\t\t\t\t * - Data Segment, pointer type\n+\t\t\t\t */\n+\t\t\t\twqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\t\t\t\tloc->wqe_last = wqe;\n+\t\t\t\tmlx5_tx_cseg_init(txq, loc, wqe, 4,\n+\t\t\t\t\t\t  MLX5_OPCODE_SEND, olx);\n+\t\t\t\tmlx5_tx_eseg_dmin(txq, loc, wqe, vlan, olx);\n+\t\t\t\tdptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) +\n+\t\t\t\t       MLX5_ESEG_MIN_INLINE_SIZE - vlan;\n+\t\t\t\t/*\n+\t\t\t\t * The length check is performed above, by\n+\t\t\t\t * comparing with txq->inlen_send. We should\n+\t\t\t\t * not get overflow here.\n+\t\t\t\t */\n+\t\t\t\tassert(inlen > MLX5_ESEG_MIN_INLINE_SIZE);\n+\t\t\t\tdlen = inlen - MLX5_ESEG_MIN_INLINE_SIZE;\n+\t\t\t\tmlx5_tx_dseg_ptr(txq, loc, &wqe->dseg[1],\n+\t\t\t\t\t\t dptr, dlen, olx);\n+\t\t\t\t++txq->wqe_ci;\n+\t\t\t\t--loc->wqe_free;\n+\t\t\t\t/* We have to store mbuf in elts.*/\n+\t\t\t\tassert(MLX5_TXOFF_CONFIG(INLINE));\n+\t\t\t\ttxq->elts[txq->elts_head++ & txq->elts_m] =\n+\t\t\t\t\t\tloc->mbuf;\n+\t\t\t\t--loc->elts_free;\n+\t\t\t}\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t\t\t/* Update sent data bytes counter. */\n+\t\t\ttxq->stats.obytes += vlan +\n+\t\t\t\t\trte_pktmbuf_data_len(loc->mbuf);\n+#endif\n+\t\t} else {\n+\t\t\t/*\n+\t\t\t * No inline at all, it means the CPU cycles saving\n+\t\t\t * is prioritized at configuration, we should not\n+\t\t\t * copy any packet data to WQE.\n+\t\t\t *\n+\t\t\t * SEND WQE, one WQEBB:\n+\t\t\t * - Control Segment, SEND opcode\n+\t\t\t * - Ethernet Segment, optional VLAN, no inline\n+\t\t\t * - Data Segment, pointer type\n+\t\t\t */\n+\t\t\twqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);\n+\t\t\tloc->wqe_last = wqe;\n+\t\t\tmlx5_tx_cseg_init(txq, loc, wqe, 3,\n+\t\t\t\t\t  MLX5_OPCODE_SEND, olx);\n+\t\t\tmlx5_tx_eseg_none(txq, loc, wqe, olx);\n+\t\t\t/*\n+\t\t\t * Some Tx offloads may cause an error if\n+\t\t\t * packet is not long enough, check against\n+\t\t\t * assumed minimal length.\n+\t\t\t */\n+\t\t\tif (rte_pktmbuf_data_len(loc->mbuf) <=\n+\t\t\t\t\t\t MLX5_ESEG_MIN_INLINE_SIZE)\n+\t\t\t\treturn MLX5_TXCMP_CODE_ERROR;\n+\t\t\tmlx5_tx_dseg_ptr\n+\t\t\t\t(txq, loc, &wqe->dseg[0],\n+\t\t\t\t rte_pktmbuf_mtod(loc->mbuf, uint8_t *),\n+\t\t\t\t rte_pktmbuf_data_len(loc->mbuf), olx);\n+\t\t\t++txq->wqe_ci;\n+\t\t\t--loc->wqe_free;\n+\t\t\t/*\n+\t\t\t * We should not store mbuf pointer in elts\n+\t\t\t * if no inlining is configured, this is done\n+\t\t\t * by calling routine in a batch copy.\n+\t\t\t */\n+\t\t\tassert(!MLX5_TXOFF_CONFIG(INLINE));\n+\t\t\t--loc->elts_free;\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t\t\t/* Update sent data bytes counter. */\n+\t\t\ttxq->stats.obytes += rte_pktmbuf_data_len(loc->mbuf);\n+\t\t\tif (MLX5_TXOFF_CONFIG(VLAN) &&\n+\t\t\t    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)\n+\t\t\t\ttxq->stats.obytes +=\n+\t\t\t\t\tsizeof(struct rte_vlan_hdr);\n+#endif\n+\t\t}\n+\t\t++loc->pkts_sent;\n+\t\t--pkts_n;\n+\t\tif (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))\n+\t\t\treturn MLX5_TXCMP_CODE_EXIT;\n+\t\tloc->mbuf = *pkts++;\n+\t\tif (pkts_n > 1)\n+\t\t\trte_prefetch0(*pkts);\n+\t\tret = mlx5_tx_able_to_empw(txq, loc, olx, true);\n+\t\tif (unlikely(ret != MLX5_TXCMP_CODE_SINGLE))\n+\t\t\treturn ret;\n+\t}\n+\tassert(false);\n+}\n+\n+static __rte_always_inline enum mlx5_txcmp_code\n+mlx5_tx_burst_single(struct mlx5_txq_data *restrict txq,\n+\t\t     struct rte_mbuf **restrict pkts,\n+\t\t     unsigned int pkts_n,\n+\t\t     struct mlx5_txq_local *restrict loc,\n+\t\t     unsigned int olx)\n+{\n+\tenum mlx5_txcmp_code ret;\n+\n+\tret = mlx5_tx_able_to_empw(txq, loc, olx, false);\n+\tif (ret == MLX5_TXCMP_CODE_SINGLE)\n+\t\tgoto ordinary_send;\n+\tassert(ret == MLX5_TXCMP_CODE_EMPW);\n+\tfor (;;) {\n+\t\t/* Optimize for inline/no inline eMPW send. */\n+\t\tret = (MLX5_TXOFF_CONFIG(INLINE)) ?\n+\t\t\tmlx5_tx_burst_empw_inline\n+\t\t\t\t(txq, pkts, pkts_n, loc, olx) :\n+\t\t\tmlx5_tx_burst_empw_simple\n+\t\t\t\t(txq, pkts, pkts_n, loc, olx);\n+\t\tif (ret != MLX5_TXCMP_CODE_SINGLE)\n+\t\t\treturn ret;\n+\t\t/* The resources to send one packet should remain. */\n+\t\tassert(loc->elts_free && loc->wqe_free);\n+ordinary_send:\n+\t\tret = mlx5_tx_burst_single_send(txq, pkts, pkts_n, loc, olx);\n+\t\tif (ret != MLX5_TXCMP_CODE_EMPW)\n+\t\t\treturn ret;\n+\t\t/* The resources to send one packet should remain. */\n+\t\tassert(loc->elts_free && loc->wqe_free);\n+\t}\n+}\n+\n+/**\n+ * DPDK Tx callback template. This is configured template\n+ * used to generate routines optimized for specified offload setup.\n+ * One of this generated functions is chosen at SQ configuration\n+ * time.\n+ *\n+ * @param txq\n+ *   Generic pointer to TX queue structure.\n+ * @param[in] pkts\n+ *   Packets to transmit.\n+ * @param pkts_n\n+ *   Number of packets in array.\n+ * @param olx\n+ *   Configured offloads mask, presents the bits of MLX5_TXOFF_CONFIG_xxx\n+ *   values. Should be static to take compile time static configuration\n+ *   advantages.\n+ *\n+ * @return\n+ *   Number of packets successfully transmitted (<= pkts_n).\n+ */\n+static __rte_always_inline uint16_t\n+mlx5_tx_burst_tmpl(struct mlx5_txq_data *restrict txq,\n+\t\t   struct rte_mbuf **restrict pkts,\n+\t\t   uint16_t pkts_n,\n+\t\t   unsigned int olx)\n+{\n+\tstruct mlx5_txq_local loc;\n+\tenum mlx5_txcmp_code ret;\n+\tunsigned int part;\n+\n+\tassert(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail));\n+\tassert(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));\n+\t/*\n+\t * Check if there are some CQEs, if any:\n+\t * - process an encountered errors\n+\t * - process the completed WQEs\n+\t * - free related mbufs\n+\t * - doorbell the NIC about processed CQEs\n+\t */\n+\tif (unlikely(!pkts_n))\n+\t\treturn 0;\n+\trte_prefetch0(*pkts);\n+\tmlx5_tx_handle_completion(txq, olx);\n+\t/*\n+\t * Calculate the number of available resources - elts and WQEs.\n+\t * There are two possible different scenarios:\n+\t * - no data inlining into WQEs, one WQEBB may contains upto\n+\t *   four packets, in this case elts become scarce resource\n+\t * - data inlining into WQEs, one packet may require multiple\n+\t *   WQEBBs, the WQEs become the limiting factor.\n+\t */\n+\tassert(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail));\n+\tloc.elts_free = txq->elts_s -\n+\t\t\t\t(uint16_t)(txq->elts_head - txq->elts_tail);\n+\tassert(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));\n+\tloc.wqe_free = txq->wqe_s -\n+\t\t\t\t(uint16_t)(txq->wqe_ci - txq->wqe_pi);\n+\tif (unlikely(!loc.elts_free || !loc.wqe_free))\n+\t\treturn 0;\n+\tloc.pkts_sent = 0;\n+\tloc.pkts_copy = 0;\n+\tloc.wqe_last = NULL;\n+\tfor (;;) {\n+\t\t/*\n+\t\t * Fetch the packet from array. Usually this is\n+\t\t * the first packet in series of multi/single\n+\t\t * segment packets.\n+\t\t */\n+\t\tloc.mbuf = *(pkts + loc.pkts_sent);\n+\t\t/* Dedicated branch for multi-segment packets. */\n+\t\tif (MLX5_TXOFF_CONFIG(MULTI) &&\n+\t\t    unlikely(NB_SEGS(loc.mbuf) > 1)) {\n+\t\t\t/*\n+\t\t\t * Multi-segment packet encountered.\n+\t\t\t * Hardware is able to process it only\n+\t\t\t * with SEND/TSO opcodes, one packet\n+\t\t\t * per WQE, do it in dedicated routine.\n+\t\t\t */\n+enter_send_multi:\n+\t\t\tassert(loc.pkts_sent >= loc.pkts_copy);\n+\t\t\tpart = loc.pkts_sent - loc.pkts_copy;\n+\t\t\tif (!MLX5_TXOFF_CONFIG(INLINE) && part) {\n+\t\t\t\t/*\n+\t\t\t\t * There are some single-segment mbufs not\n+\t\t\t\t * stored in elts. The mbufs must be in the\n+\t\t\t\t * same order as WQEs, so we must copy the\n+\t\t\t\t * mbufs to elts here, before the coming\n+\t\t\t\t * multi-segment packet mbufs is appended.\n+\t\t\t\t */\n+\t\t\t\tmlx5_tx_copy_elts(txq, pkts + loc.pkts_copy,\n+\t\t\t\t\t\t  part, olx);\n+\t\t\t\tloc.pkts_copy = loc.pkts_sent;\n+\t\t\t}\n+\t\t\tassert(pkts_n > loc.pkts_sent);\n+\t\t\tret = mlx5_tx_burst_mseg(txq, pkts, pkts_n, &loc, olx);\n+\t\t\tif (!MLX5_TXOFF_CONFIG(INLINE))\n+\t\t\t\tloc.pkts_copy = loc.pkts_sent;\n+\t\t\t/*\n+\t\t\t * These returned code checks are supposed\n+\t\t\t * to be optimized out due to routine inlining.\n+\t\t\t */\n+\t\t\tif (ret == MLX5_TXCMP_CODE_EXIT) {\n+\t\t\t\t/*\n+\t\t\t\t * The routine returns this code when\n+\t\t\t\t * all packets are sent or there is no\n+\t\t\t\t * enough resources to complete request.\n+\t\t\t\t */\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tif (ret == MLX5_TXCMP_CODE_ERROR) {\n+\t\t\t\t/*\n+\t\t\t\t * The routine returns this code when\n+\t\t\t\t * some error in the incoming packets\n+\t\t\t\t * format occurred.\n+\t\t\t\t */\n+\t\t\t\ttxq->stats.oerrors++;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tif (ret == MLX5_TXCMP_CODE_SINGLE) {\n+\t\t\t\t/*\n+\t\t\t\t * The single-segment packet was encountered\n+\t\t\t\t * in the array, try to send it with the\n+\t\t\t\t * best optimized way, possible engaging eMPW.\n+\t\t\t\t */\n+\t\t\t\tgoto enter_send_single;\n+\t\t\t}\n+\t\t\tif (MLX5_TXOFF_CONFIG(TSO) &&\n+\t\t\t    ret == MLX5_TXCMP_CODE_TSO) {\n+\t\t\t\t/*\n+\t\t\t\t * The single-segment TSO packet was\n+\t\t\t\t * encountered in the array.\n+\t\t\t\t */\n+\t\t\t\tgoto enter_send_tso;\n+\t\t\t}\n+\t\t\t/* We must not get here. Something is going wrong. */\n+\t\t\tassert(false);\n+\t\t\ttxq->stats.oerrors++;\n+\t\t\tbreak;\n+\t\t}\n+\t\t/* Dedicated branch for single-segment TSO packets. */\n+\t\tif (MLX5_TXOFF_CONFIG(TSO) &&\n+\t\t    unlikely(loc.mbuf->ol_flags & PKT_TX_TCP_SEG)) {\n+\t\t\t/*\n+\t\t\t * TSO might require special way for inlining\n+\t\t\t * (dedicated parameters) and is sent with\n+\t\t\t * MLX5_OPCODE_TSO opcode only, provide this\n+\t\t\t * in dedicated branch.\n+\t\t\t */\n+enter_send_tso:\n+\t\t\tassert(NB_SEGS(loc.mbuf) == 1);\n+\t\t\tassert(pkts_n > loc.pkts_sent);\n+\t\t\tret = mlx5_tx_burst_tso(txq, pkts, pkts_n, &loc, olx);\n+\t\t\t/*\n+\t\t\t * These returned code checks are supposed\n+\t\t\t * to be optimized out due to routine inlining.\n+\t\t\t */\n+\t\t\tif (ret == MLX5_TXCMP_CODE_EXIT)\n+\t\t\t\tbreak;\n+\t\t\tif (ret == MLX5_TXCMP_CODE_ERROR) {\n+\t\t\t\ttxq->stats.oerrors++;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tif (ret == MLX5_TXCMP_CODE_SINGLE)\n+\t\t\t\tgoto enter_send_single;\n+\t\t\tif (MLX5_TXOFF_CONFIG(MULTI) &&\n+\t\t\t    ret == MLX5_TXCMP_CODE_MULTI) {\n+\t\t\t\t/*\n+\t\t\t\t * The multi-segment packet was\n+\t\t\t\t * encountered in the array.\n+\t\t\t\t */\n+\t\t\t\tgoto enter_send_multi;\n+\t\t\t}\n+\t\t\t/* We must not get here. Something is going wrong. */\n+\t\t\tassert(false);\n+\t\t\ttxq->stats.oerrors++;\n+\t\t\tbreak;\n+\t\t}\n+\t\t/*\n+\t\t * The dedicated branch for the single-segment packets\n+\t\t * without TSO. Often these ones can be sent using\n+\t\t * MLX5_OPCODE_EMPW with multiple packets in one WQE.\n+\t\t * The routine builds the WQEs till it encounters\n+\t\t * the TSO or multi-segment packet (in case if these\n+\t\t * offloads are requested at SQ configuration time).\n+\t\t */\n+enter_send_single:\n+\t\tassert(pkts_n > loc.pkts_sent);\n+\t\tret = mlx5_tx_burst_single(txq, pkts, pkts_n, &loc, olx);\n+\t\t/*\n+\t\t * These returned code checks are supposed\n+\t\t * to be optimized out due to routine inlining.\n+\t\t */\n+\t\tif (ret == MLX5_TXCMP_CODE_EXIT)\n+\t\t\tbreak;\n+\t\tif (ret == MLX5_TXCMP_CODE_ERROR) {\n+\t\t\ttxq->stats.oerrors++;\n+\t\t\tbreak;\n+\t\t}\n+\t\tif (ret == MLX5_TXCMP_CODE_SINGLE)\n+\t\t\tgoto enter_send_single;\n+\t\tif (MLX5_TXOFF_CONFIG(MULTI) &&\n+\t\t    ret == MLX5_TXCMP_CODE_MULTI)\n+\t\t\tgoto enter_send_multi;\n+\t\t/* We must not get here. Something is going wrong. */\n+\t\tassert(false);\n+\t\ttxq->stats.oerrors++;\n+\t\tbreak;\n+\t}\n+\t/*\n+\t * Main Tx loop is completed, do the rest:\n+\t * - set completion request if thresholds are reached\n+\t * - doorbell the hardware\n+\t * - copy the rest of mbufs to elts (if any)\n+\t */\n+\tassert(MLX5_TXOFF_CONFIG(INLINE) || loc.pkts_sent >= loc.pkts_copy);\n+\t/* Take a shortcut if nothing is sent. */\n+\tif (unlikely(loc.pkts_sent == 0))\n+\t\treturn 0;\n+\t/* Not all of the mbufs may be stored into elts yet. */\n+\tpart = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc.pkts_sent - loc.pkts_copy;\n+\tmlx5_tx_request_completion(txq, part, &loc, olx);\n+\t/*\n+\t * Ring QP doorbell immediately after WQE building completion\n+\t * to improve latencies. The pure software related data treatment\n+\t * can be completed after doorbell. Tx CQEs for this SQ are\n+\t * processed in this thread only by the polling.\n+\t */\n+\tmlx5_tx_dbrec_cond_wmb(txq, loc.wqe_last, 0);\n+\tif (!MLX5_TXOFF_CONFIG(INLINE) && part) {\n+\t\t/*\n+\t\t * There are some single-segment mbufs not stored in elts.\n+\t\t * It can be only if last packet was single-segment.\n+\t\t * The copying is gathered into one place due to it is\n+\t\t * a good opportunity to optimize that with SIMD.\n+\t\t * Unfortunately if inlining is enabled the gaps in\n+\t\t * pointer array may happen due to early freeing of the\n+\t\t * inlined mbufs.\n+\t\t */\n+\t\tmlx5_tx_copy_elts(txq, pkts + loc.pkts_copy, part, olx);\n+\t}\n+#ifdef MLX5_PMD_SOFT_COUNTERS\n+\t/* Increment sent packets counter. */\n+\ttxq->stats.opackets += loc.pkts_sent;\n+#endif\n+\tassert(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail));\n+\tassert(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));\n+\treturn loc.pkts_sent;\n }\n \n /* Generate routines with Enhanced Multi-Packet Write support. */\ndiff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h\nindex d8c6f35..4439e88 100644\n--- a/drivers/net/mlx5/mlx5_rxtx.h\n+++ b/drivers/net/mlx5/mlx5_rxtx.h\n@@ -354,8 +354,9 @@ struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,\n void mlx5_set_ptype_table(void);\n void mlx5_set_cksum_table(void);\n void mlx5_set_swp_types_table(void);\n-__rte_noinline uint16_t mlx5_tx_error_cqe_handle(struct mlx5_txq_data *txq,\n-\t\t\t\t\tvolatile struct mlx5_err_cqe *err_cqe);\n+__rte_noinline uint16_t mlx5_tx_error_cqe_handle\n+\t\t\t\t(struct mlx5_txq_data *restrict txq,\n+\t\t\t\t volatile struct mlx5_err_cqe *err_cqe);\n uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n);\n void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq);\n __rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq,\n",
    "prefixes": [
        "v2",
        "6/7"
    ]
}