Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 136525,
    "url": "http://patches.dpdk.org/api/patches/136525/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/20240208085956.1741174-10-ndabilpuram@marvell.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20240208085956.1741174-10-ndabilpuram@marvell.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20240208085956.1741174-10-ndabilpuram@marvell.com",
    "date": "2024-02-08T08:59:53",
    "name": "[10/13] net/cnxk: fix indirect mbuf handling in Tx path",
    "commit_ref": null,
    "pull_url": null,
    "state": "changes-requested",
    "archived": true,
    "hash": "db0da0281fde8c3c597a58a78839afac3f897d06",
    "submitter": {
        "id": 1202,
        "url": "http://patches.dpdk.org/api/people/1202/?format=api",
        "name": "Nithin Dabilpuram",
        "email": "ndabilpuram@marvell.com"
    },
    "delegate": {
        "id": 310,
        "url": "http://patches.dpdk.org/api/users/310/?format=api",
        "username": "jerin",
        "first_name": "Jerin",
        "last_name": "Jacob",
        "email": "jerinj@marvell.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/20240208085956.1741174-10-ndabilpuram@marvell.com/mbox/",
    "series": [
        {
            "id": 31048,
            "url": "http://patches.dpdk.org/api/series/31048/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=31048",
            "date": "2024-02-08T08:59:44",
            "name": "[01/13] common/cnxk: remove cn9k Inline IPsec FP opcode defines",
            "version": 1,
            "mbox": "http://patches.dpdk.org/series/31048/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/136525/comments/",
    "check": "warning",
    "checks": "http://patches.dpdk.org/api/patches/136525/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 751B443ACC;\n\tThu,  8 Feb 2024 10:01:37 +0100 (CET)",
            "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 782D442E54;\n\tThu,  8 Feb 2024 10:00:31 +0100 (CET)",
            "from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com\n [67.231.156.173])\n by mails.dpdk.org (Postfix) with ESMTP id 621E042E30\n for <dev@dpdk.org>; Thu,  8 Feb 2024 10:00:28 +0100 (CET)",
            "from pps.filterd (m0045851.ppops.net [127.0.0.1])\n by mx0b-0016f401.pphosted.com (8.17.1.24/8.17.1.24) with ESMTP id\n 4184GWlb000449 for <dev@dpdk.org>; Thu, 8 Feb 2024 01:00:27 -0800",
            "from dc5-exch01.marvell.com ([199.233.59.181])\n by mx0b-0016f401.pphosted.com (PPS) with ESMTPS id 3w4qsq0pv6-1\n (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT)\n for <dev@dpdk.org>; Thu, 08 Feb 2024 01:00:27 -0800 (PST)",
            "from DC5-EXCH01.marvell.com (10.69.176.38) by DC5-EXCH01.marvell.com\n (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.48;\n Thu, 8 Feb 2024 01:00:25 -0800",
            "from maili.marvell.com (10.69.176.80) by DC5-EXCH01.marvell.com\n (10.69.176.38) with Microsoft SMTP Server id 15.0.1497.48 via Frontend\n Transport; Thu, 8 Feb 2024 01:00:25 -0800",
            "from hyd1588t430.caveonetworks.com (unknown [10.29.52.204])\n by maili.marvell.com (Postfix) with ESMTP id 98C813F7090;\n Thu,  8 Feb 2024 01:00:23 -0800 (PST)"
        ],
        "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h=\n from:to:cc:subject:date:message-id:in-reply-to:references\n :mime-version:content-transfer-encoding:content-type; s=\n pfpt0220; bh=fHWA4yPrmkKAO4LbNcG8XpmTTNvfRlS3vMK5wb2b8NI=; b=LIK\n SkZxHezrLPqmFclygrmeP61lIzskmtbzuIjnhdJOef8bwipLOvCqj+8EUGXqyobA\n xQTuynzQFwH21GzwIbo7EQM0xBCX8FBcD0jB8XKADV2OrIF1e4ivne0Hz3KTnQcH\n w7hBJKze4ZaSF5c/XOr4dOBnj/ojjP5aowgmMH7pkBB88IRWGkcJ4Su0QBMrkWPs\n 6JMMsJcm9AiINrW1kDOUWhE9lENKY8kJDH3hJefgeiJtJkYjBWsMh0CgwN6/gRcW\n cVdh/wrFyzKpQeswfPktOLoXl1ojZE3auJC3ctvfCrvs4Pcb+ZlVNiFp/Ti1+4zv\n u4JPc0Ql4Agw+StwoNw==",
        "From": "Nithin Dabilpuram <ndabilpuram@marvell.com>",
        "To": "Nithin Dabilpuram <ndabilpuram@marvell.com>, Kiran Kumar K\n <kirankumark@marvell.com>, Sunil Kumar Kori <skori@marvell.com>, Satha Rao\n <skoteshwar@marvell.com>",
        "CC": "<dev@dpdk.org>",
        "Subject": "[PATCH 10/13] net/cnxk: fix indirect mbuf handling in Tx path",
        "Date": "Thu, 8 Feb 2024 14:29:53 +0530",
        "Message-ID": "<20240208085956.1741174-10-ndabilpuram@marvell.com>",
        "X-Mailer": "git-send-email 2.25.1",
        "In-Reply-To": "<20240208085956.1741174-1-ndabilpuram@marvell.com>",
        "References": "<20240208085956.1741174-1-ndabilpuram@marvell.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "Content-Type": "text/plain",
        "X-Proofpoint-ORIG-GUID": "twBM8MbCbApdffVkD6QCMmFkuOlOH38Q",
        "X-Proofpoint-GUID": "twBM8MbCbApdffVkD6QCMmFkuOlOH38Q",
        "X-Proofpoint-Virus-Version": "vendor=baseguard\n engine=ICAP:2.0.272,Aquarius:18.0.1011,Hydra:6.0.619,FMLib:17.11.176.26\n definitions=2024-02-08_01,2024-02-07_01,2023-05-22_02",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org"
    },
    "content": "Indirect mbuf can be pointing to data from different pool. Use the right\naura in NIX send header in SG2 and SG case.\n\nFixes: 862e28128707 (\"net/cnxk: add vector Tx for CN9K\")\nFixes: f71b7dbbf04b (\"net/cnxk: add vector Tx for CN10K\")\nFixes: 7e95c11df4f1 (\"net/cnxk: add multi-segment Tx for CN9K\")\nFixes: 3626d5195d49 (\"net/cnxk: add multi-segment Tx for CN10K\")\n\nSigned-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>\n---\n drivers/net/cnxk/cn10k_ethdev.c   |   6 +\n drivers/net/cnxk/cn10k_rxtx.h     |   1 +\n drivers/net/cnxk/cn10k_tx.h       | 265 ++++++++++++++++++---------\n drivers/net/cnxk/cn9k_ethdev.c    |   6 +\n drivers/net/cnxk/cn9k_ethdev.h    |   1 +\n drivers/net/cnxk/cn9k_tx.h        | 288 +++++++++++++++++++++---------\n drivers/net/cnxk/cnxk_ethdev_dp.h |  10 +-\n 7 files changed, 402 insertions(+), 175 deletions(-)",
    "diff": "diff --git a/drivers/net/cnxk/cn10k_ethdev.c b/drivers/net/cnxk/cn10k_ethdev.c\nindex a2e943a3d0..a5696c092a 100644\n--- a/drivers/net/cnxk/cn10k_ethdev.c\n+++ b/drivers/net/cnxk/cn10k_ethdev.c\n@@ -389,7 +389,13 @@ cn10k_nix_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t qidx)\n \t\tstruct roc_nix_sq *sq = &dev->sqs[qidx];\n \t\tdo {\n \t\t\thandle_tx_completion_pkts(txq, flags & NIX_TX_VWQE_F);\n+\t\t\t/* Check if SQ is empty */\n \t\t\troc_nix_sq_head_tail_get(nix, sq->qid, &head, &tail);\n+\t\t\tif (head != tail)\n+\t\t\t\tcontinue;\n+\n+\t\t\t/* Check if completion CQ is empty */\n+\t\t\troc_nix_cq_head_tail_get(nix, sq->cqid, &head, &tail);\n \t\t} while (head != tail);\n \t}\n \ndiff --git a/drivers/net/cnxk/cn10k_rxtx.h b/drivers/net/cnxk/cn10k_rxtx.h\nindex aeffc4ac92..9f33d0192e 100644\n--- a/drivers/net/cnxk/cn10k_rxtx.h\n+++ b/drivers/net/cnxk/cn10k_rxtx.h\n@@ -177,6 +177,7 @@ handle_tx_completion_pkts(struct cn10k_eth_txq *txq, uint8_t mt_safe)\n \t\t\tm = m_next;\n \t\t}\n \t\trte_pktmbuf_free_seg(m);\n+\t\ttxq->tx_compl.ptr[tx_compl_s0->sqe_id] = NULL;\n \n \t\thead++;\n \t\thead &= qmask;\ndiff --git a/drivers/net/cnxk/cn10k_tx.h b/drivers/net/cnxk/cn10k_tx.h\nindex 467f0ccc65..9721b7584a 100644\n--- a/drivers/net/cnxk/cn10k_tx.h\n+++ b/drivers/net/cnxk/cn10k_tx.h\n@@ -786,8 +786,9 @@ cn10k_nix_prep_sec(struct rte_mbuf *m, uint64_t *cmd, uintptr_t *nixtx_addr,\n \n static __rte_always_inline uint64_t\n cn10k_nix_prefree_seg(struct rte_mbuf *m, struct cn10k_eth_txq *txq,\n-\t\tstruct nix_send_hdr_s *send_hdr)\n+\t\t      struct nix_send_hdr_s *send_hdr, uint64_t *aura)\n {\n+\tstruct rte_mbuf *prev = NULL;\n \tuint32_t sqe_id;\n \n \tif (RTE_MBUF_HAS_EXTBUF(m)) {\n@@ -796,7 +797,10 @@ cn10k_nix_prefree_seg(struct rte_mbuf *m, struct cn10k_eth_txq *txq,\n \t\t\treturn 1;\n \t\t}\n \t\tif (send_hdr->w0.pnc) {\n-\t\t\ttxq->tx_compl.ptr[send_hdr->w1.sqe_id]->next = m;\n+\t\t\tsqe_id = send_hdr->w1.sqe_id;\n+\t\t\tprev = txq->tx_compl.ptr[sqe_id];\n+\t\t\tm->next = prev;\n+\t\t\ttxq->tx_compl.ptr[sqe_id] = m;\n \t\t} else {\n \t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n \t\t\tsend_hdr->w0.pnc = 1;\n@@ -806,10 +810,151 @@ cn10k_nix_prefree_seg(struct rte_mbuf *m, struct cn10k_eth_txq *txq,\n \t\t}\n \t\treturn 1;\n \t} else {\n-\t\treturn cnxk_nix_prefree_seg(m);\n+\t\treturn cnxk_nix_prefree_seg(m, aura);\n \t}\n }\n \n+#if defined(RTE_ARCH_ARM64)\n+/* Only called for first segments of single segmented mbufs */\n+static __rte_always_inline void\n+cn10k_nix_prefree_seg_vec(struct rte_mbuf **mbufs, struct cn10k_eth_txq *txq,\n+\t\t\t  uint64x2_t *senddesc01_w0, uint64x2_t *senddesc23_w0,\n+\t\t\t  uint64x2_t *senddesc01_w1, uint64x2_t *senddesc23_w1)\n+{\n+\tstruct rte_mbuf **tx_compl_ptr = txq->tx_compl.ptr;\n+\tuint32_t nb_desc_mask = txq->tx_compl.nb_desc_mask;\n+\tbool tx_compl_ena = txq->tx_compl.ena;\n+\tstruct rte_mbuf *m0, *m1, *m2, *m3;\n+\tstruct rte_mbuf *cookie;\n+\tuint64_t w0, w1, aura;\n+\tuint64_t sqe_id;\n+\n+\tm0 = mbufs[0];\n+\tm1 = mbufs[1];\n+\tm2 = mbufs[2];\n+\tm3 = mbufs[3];\n+\n+\t/* mbuf 0 */\n+\tw0 = vgetq_lane_u64(*senddesc01_w0, 0);\n+\tif (RTE_MBUF_HAS_EXTBUF(m0)) {\n+\t\tw0 |= BIT_ULL(19);\n+\t\tw1 = vgetq_lane_u64(*senddesc01_w1, 0);\n+\t\tw1 &= ~0xFFFF000000000000UL;\n+\t\tif (unlikely(!tx_compl_ena)) {\n+\t\t\trte_pktmbuf_free_seg(m0);\n+\t\t} else {\n+\t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n+\t\t\tsqe_id = sqe_id & nb_desc_mask;\n+\t\t\t/* Set PNC */\n+\t\t\tw0 |= BIT_ULL(43);\n+\t\t\tw1 |= sqe_id << 48;\n+\t\t\ttx_compl_ptr[sqe_id] = m0;\n+\t\t\t*senddesc01_w1 = vsetq_lane_u64(w1, *senddesc01_w1, 0);\n+\t\t}\n+\t} else {\n+\t\tcookie = RTE_MBUF_DIRECT(m0) ? m0 : rte_mbuf_from_indirect(m0);\n+\t\taura = (w0 >> 20) & 0xFFFFF;\n+\t\tw0 &= ~0xFFFFF00000UL;\n+\t\tw0 |= cnxk_nix_prefree_seg(m0, &aura) << 19;\n+\t\tw0 |= aura << 20;\n+\n+\t\tif ((w0 & BIT_ULL(19)) == 0)\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, &cookie, 1, 0);\n+\t}\n+\t*senddesc01_w0 = vsetq_lane_u64(w0, *senddesc01_w0, 0);\n+\n+\t/* mbuf1 */\n+\tw0 = vgetq_lane_u64(*senddesc01_w0, 1);\n+\tif (RTE_MBUF_HAS_EXTBUF(m1)) {\n+\t\tw0 |= BIT_ULL(19);\n+\t\tw1 = vgetq_lane_u64(*senddesc01_w1, 1);\n+\t\tw1 &= ~0xFFFF000000000000UL;\n+\t\tif (unlikely(!tx_compl_ena)) {\n+\t\t\trte_pktmbuf_free_seg(m1);\n+\t\t} else {\n+\t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n+\t\t\tsqe_id = sqe_id & nb_desc_mask;\n+\t\t\t/* Set PNC */\n+\t\t\tw0 |= BIT_ULL(43);\n+\t\t\tw1 |= sqe_id << 48;\n+\t\t\ttx_compl_ptr[sqe_id] = m1;\n+\t\t\t*senddesc01_w1 = vsetq_lane_u64(w1, *senddesc01_w1, 1);\n+\t\t}\n+\t} else {\n+\t\tcookie = RTE_MBUF_DIRECT(m1) ? m1 : rte_mbuf_from_indirect(m1);\n+\t\taura = (w0 >> 20) & 0xFFFFF;\n+\t\tw0 &= ~0xFFFFF00000UL;\n+\t\tw0 |= cnxk_nix_prefree_seg(m1, &aura) << 19;\n+\t\tw0 |= aura << 20;\n+\n+\t\tif ((w0 & BIT_ULL(19)) == 0)\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, &cookie, 1, 0);\n+\t}\n+\t*senddesc01_w0 = vsetq_lane_u64(w0, *senddesc01_w0, 1);\n+\n+\t/* mbuf 2 */\n+\tw0 = vgetq_lane_u64(*senddesc23_w0, 0);\n+\tif (RTE_MBUF_HAS_EXTBUF(m2)) {\n+\t\tw0 |= BIT_ULL(19);\n+\t\tw1 = vgetq_lane_u64(*senddesc23_w1, 0);\n+\t\tw1 &= ~0xFFFF000000000000UL;\n+\t\tif (unlikely(!tx_compl_ena)) {\n+\t\t\trte_pktmbuf_free_seg(m2);\n+\t\t} else {\n+\t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n+\t\t\tsqe_id = sqe_id & nb_desc_mask;\n+\t\t\t/* Set PNC */\n+\t\t\tw0 |= BIT_ULL(43);\n+\t\t\tw1 |= sqe_id << 48;\n+\t\t\ttx_compl_ptr[sqe_id] = m2;\n+\t\t\t*senddesc23_w1 = vsetq_lane_u64(w1, *senddesc23_w1, 0);\n+\t\t}\n+\t} else {\n+\t\tcookie = RTE_MBUF_DIRECT(m2) ? m2 : rte_mbuf_from_indirect(m2);\n+\t\taura = (w0 >> 20) & 0xFFFFF;\n+\t\tw0 &= ~0xFFFFF00000UL;\n+\t\tw0 |= cnxk_nix_prefree_seg(m2, &aura) << 19;\n+\t\tw0 |= aura << 20;\n+\n+\t\tif ((w0 & BIT_ULL(19)) == 0)\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, &cookie, 1, 0);\n+\t}\n+\t*senddesc23_w0 = vsetq_lane_u64(w0, *senddesc23_w0, 0);\n+\n+\t/* mbuf3 */\n+\tw0 = vgetq_lane_u64(*senddesc23_w0, 1);\n+\tif (RTE_MBUF_HAS_EXTBUF(m3)) {\n+\t\tw0 |= BIT_ULL(19);\n+\t\tw1 = vgetq_lane_u64(*senddesc23_w1, 1);\n+\t\tw1 &= ~0xFFFF000000000000UL;\n+\t\tif (unlikely(!tx_compl_ena)) {\n+\t\t\trte_pktmbuf_free_seg(m3);\n+\t\t} else {\n+\t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n+\t\t\tsqe_id = sqe_id & nb_desc_mask;\n+\t\t\t/* Set PNC */\n+\t\t\tw0 |= BIT_ULL(43);\n+\t\t\tw1 |= sqe_id << 48;\n+\t\t\ttx_compl_ptr[sqe_id] = m3;\n+\t\t\t*senddesc23_w1 = vsetq_lane_u64(w1, *senddesc23_w1, 1);\n+\t\t}\n+\t} else {\n+\t\tcookie = RTE_MBUF_DIRECT(m3) ? m3 : rte_mbuf_from_indirect(m3);\n+\t\taura = (w0 >> 20) & 0xFFFFF;\n+\t\tw0 &= ~0xFFFFF00000UL;\n+\t\tw0 |= cnxk_nix_prefree_seg(m3, &aura) << 19;\n+\t\tw0 |= aura << 20;\n+\n+\t\tif ((w0 & BIT_ULL(19)) == 0)\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, &cookie, 1, 0);\n+\t}\n+\t*senddesc23_w0 = vsetq_lane_u64(w0, *senddesc23_w0, 1);\n+#ifndef RTE_LIBRTE_MEMPOOL_DEBUG\n+\tRTE_SET_USED(cookie);\n+#endif\n+}\n+#endif\n+\n static __rte_always_inline void\n cn10k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags)\n {\n@@ -889,6 +1034,9 @@ cn10k_nix_xmit_prepare(struct cn10k_eth_txq *txq,\n \t\tsg = (union nix_send_sg_s *)(cmd + 2);\n \t}\n \n+\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)\n+\t\tsend_hdr->w0.pnc = 0;\n+\n \tif (flags & (NIX_TX_NEED_SEND_HDR_W1 | NIX_TX_OFFLOAD_SECURITY_F)) {\n \t\tol_flags = m->ol_flags;\n \t\tw1.u = 0;\n@@ -1049,19 +1197,30 @@ cn10k_nix_xmit_prepare(struct cn10k_eth_txq *txq,\n \t\tsend_hdr->w1.u = w1.u;\n \n \tif (!(flags & NIX_TX_MULTI_SEG_F)) {\n+\t\tstruct rte_mbuf *cookie;\n+\n \t\tsg->seg1_size = send_hdr->w0.total;\n \t\t*(rte_iova_t *)(sg + 1) = rte_mbuf_data_iova(m);\n+\t\tcookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m);\n \n \t\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {\n+\t\t\tuint64_t aura;\n+\n \t\t\t/* DF bit = 1 if refcount of current mbuf or parent mbuf\n \t\t\t *\t\tis greater than 1\n \t\t\t * DF bit = 0 otherwise\n \t\t\t */\n-\t\t\tsend_hdr->w0.df = cn10k_nix_prefree_seg(m, txq, send_hdr);\n+\t\t\taura = send_hdr->w0.aura;\n+\t\t\tsend_hdr->w0.df = cn10k_nix_prefree_seg(m, txq, send_hdr, &aura);\n+\t\t\tsend_hdr->w0.aura = aura;\n \t\t}\n+#ifdef RTE_LIBRTE_MEMPOOL_DEBUG\n \t\t/* Mark mempool object as \"put\" since it is freed by NIX */\n \t\tif (!send_hdr->w0.df)\n-\t\t\tRTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0);\n+#else\n+\t\tRTE_SET_USED(cookie);\n+#endif\n \t} else {\n \t\tsg->seg1_size = m->data_len;\n \t\t*(rte_iova_t *)(sg + 1) = rte_mbuf_data_iova(m);\n@@ -1135,6 +1294,7 @@ cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq,\n \tstruct nix_send_hdr_s *send_hdr;\n \tunion nix_send_sg_s *sg, l_sg;\n \tunion nix_send_sg2_s l_sg2;\n+\tstruct rte_mbuf *cookie;\n \tstruct rte_mbuf *m_next;\n \tuint8_t off, is_sg2;\n \tuint64_t len, dlen;\n@@ -1163,21 +1323,26 @@ cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq,\n \tlen -= dlen;\n \tnb_segs = m->nb_segs - 1;\n \tm_next = m->next;\n+\tm->next = NULL;\n \tslist = &cmd[3 + off + 1];\n \n+\tcookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m);\n \t/* Set invert df if buffer is not to be freed by H/W */\n \tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {\n-\t\tprefree = cn10k_nix_prefree_seg(m, txq, send_hdr);\n+\t\taura = send_hdr->w0.aura;\n+\t\tprefree = cn10k_nix_prefree_seg(m, txq, send_hdr, &aura);\n+\t\tsend_hdr->w0.aura = aura;\n \t\tl_sg.i1 = prefree;\n \t}\n \n #ifdef RTE_LIBRTE_MEMPOOL_DEBUG\n \t/* Mark mempool object as \"put\" since it is freed by NIX */\n \tif (!prefree)\n-\t\tRTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);\n+\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0);\n \trte_io_wmb();\n+#else\n+\tRTE_SET_USED(cookie);\n #endif\n-\tm->next = NULL;\n \n \t/* Quickly handle single segmented packets. With this if-condition\n \t * compiler will completely optimize out the below do-while loop\n@@ -1207,9 +1372,12 @@ cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq,\n \t\taura = aura0;\n \t\tprefree = 0;\n \n+\t\tm->next = NULL;\n+\n+\t\tcookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m);\n \t\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {\n \t\t\taura = roc_npa_aura_handle_to_aura(m->pool->pool_id);\n-\t\t\tprefree = cn10k_nix_prefree_seg(m, txq, send_hdr);\n+\t\t\tprefree = cn10k_nix_prefree_seg(m, txq, send_hdr, &aura);\n \t\t\tis_sg2 = aura != aura0 && !prefree;\n \t\t}\n \n@@ -1259,13 +1427,14 @@ cn10k_nix_prepare_mseg(struct cn10k_eth_txq *txq,\n \t\t\tl_sg.subdc = NIX_SUBDC_SG;\n \t\t\tslist++;\n \t\t}\n-\t\tm->next = NULL;\n \n #ifdef RTE_LIBRTE_MEMPOOL_DEBUG\n \t\t/* Mark mempool object as \"put\" since it is freed by NIX\n \t\t */\n \t\tif (!prefree)\n-\t\t\tRTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0);\n+#else\n+\t\tRTE_SET_USED(cookie);\n #endif\n \t\tm = m_next;\n \t} while (nb_segs);\n@@ -1997,13 +2166,10 @@ cn10k_nix_xmit_pkts_vector(void *tx_queue, uint64_t *ws,\n \tuint64x2_t sgdesc01_w0, sgdesc23_w0;\n \tuint64x2_t sgdesc01_w1, sgdesc23_w1;\n \tstruct cn10k_eth_txq *txq = tx_queue;\n-\tuint64x2_t xmask01_w0, xmask23_w0;\n-\tuint64x2_t xmask01_w1, xmask23_w1;\n \trte_iova_t io_addr = txq->io_addr;\n \tuint8_t lnum, shift = 0, loff = 0;\n \tuintptr_t laddr = txq->lmt_base;\n \tuint8_t c_lnum, c_shft, c_loff;\n-\tstruct nix_send_hdr_s send_hdr;\n \tuint64x2_t ltypes01, ltypes23;\n \tuint64x2_t xtmp128, ytmp128;\n \tuint64x2_t xmask01, xmask23;\n@@ -2153,7 +2319,7 @@ cn10k_nix_xmit_pkts_vector(void *tx_queue, uint64_t *ws,\n \t\t}\n \t\t/* Clear lower 32bit of SEND_HDR_W0 and SEND_SG_W0 */\n \t\tsenddesc01_w0 =\n-\t\t\tvbicq_u64(senddesc01_w0, vdupq_n_u64(0xFFFFFFFF));\n+\t\t\tvbicq_u64(senddesc01_w0, vdupq_n_u64(0x800FFFFFFFF));\n \t\tsgdesc01_w0 = vbicq_u64(sgdesc01_w0, vdupq_n_u64(0xFFFFFFFF));\n \n \t\tsenddesc23_w0 = senddesc01_w0;\n@@ -2859,73 +3025,8 @@ cn10k_nix_xmit_pkts_vector(void *tx_queue, uint64_t *ws,\n \t\t    !(flags & NIX_TX_MULTI_SEG_F) &&\n \t\t    !(flags & NIX_TX_OFFLOAD_SECURITY_F)) {\n \t\t\t/* Set don't free bit if reference count > 1 */\n-\t\t\txmask01_w0 = vdupq_n_u64(0);\n-\t\t\txmask01_w1 = vdupq_n_u64(0);\n-\t\t\txmask23_w0 = xmask01_w0;\n-\t\t\txmask23_w1 = xmask01_w1;\n-\n-\t\t\t/* Move mbufs to iova */\n-\t\t\tmbuf0 = (uint64_t *)tx_pkts[0];\n-\t\t\tmbuf1 = (uint64_t *)tx_pkts[1];\n-\t\t\tmbuf2 = (uint64_t *)tx_pkts[2];\n-\t\t\tmbuf3 = (uint64_t *)tx_pkts[3];\n-\n-\t\t\tsend_hdr.w0.u = 0;\n-\t\t\tsend_hdr.w1.u = 0;\n-\n-\t\t\tif (cn10k_nix_prefree_seg((struct rte_mbuf *)mbuf0, txq, &send_hdr)) {\n-\t\t\t\tsend_hdr.w0.df = 1;\n-\t\t\t\txmask01_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask01_w0, 0);\n-\t\t\t\txmask01_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask01_w1, 0);\n-\t\t\t} else {\n-\t\t\t\tRTE_MEMPOOL_CHECK_COOKIES(\n-\t\t\t\t\t((struct rte_mbuf *)mbuf0)->pool,\n-\t\t\t\t\t(void **)&mbuf0, 1, 0);\n-\t\t\t}\n-\n-\t\t\tsend_hdr.w0.u = 0;\n-\t\t\tsend_hdr.w1.u = 0;\n-\n-\t\t\tif (cn10k_nix_prefree_seg((struct rte_mbuf *)mbuf1, txq, &send_hdr)) {\n-\t\t\t\tsend_hdr.w0.df = 1;\n-\t\t\t\txmask01_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask01_w0, 1);\n-\t\t\t\txmask01_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask01_w1, 1);\n-\t\t\t} else {\n-\t\t\t\tRTE_MEMPOOL_CHECK_COOKIES(\n-\t\t\t\t\t((struct rte_mbuf *)mbuf1)->pool,\n-\t\t\t\t\t(void **)&mbuf1, 1, 0);\n-\t\t\t}\n-\n-\t\t\tsend_hdr.w0.u = 0;\n-\t\t\tsend_hdr.w1.u = 0;\n-\n-\t\t\tif (cn10k_nix_prefree_seg((struct rte_mbuf *)mbuf2, txq, &send_hdr)) {\n-\t\t\t\tsend_hdr.w0.df = 1;\n-\t\t\t\txmask23_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask23_w0, 0);\n-\t\t\t\txmask23_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask23_w1, 0);\n-\t\t\t} else {\n-\t\t\t\tRTE_MEMPOOL_CHECK_COOKIES(\n-\t\t\t\t\t((struct rte_mbuf *)mbuf2)->pool,\n-\t\t\t\t\t(void **)&mbuf2, 1, 0);\n-\t\t\t}\n-\n-\t\t\tsend_hdr.w0.u = 0;\n-\t\t\tsend_hdr.w1.u = 0;\n-\n-\t\t\tif (cn10k_nix_prefree_seg((struct rte_mbuf *)mbuf3, txq, &send_hdr)) {\n-\t\t\t\tsend_hdr.w0.df = 1;\n-\t\t\t\txmask23_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask23_w0, 1);\n-\t\t\t\txmask23_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask23_w1, 1);\n-\t\t\t} else {\n-\t\t\t\tRTE_MEMPOOL_CHECK_COOKIES(\n-\t\t\t\t\t((struct rte_mbuf *)mbuf3)->pool,\n-\t\t\t\t\t(void **)&mbuf3, 1, 0);\n-\t\t\t}\n-\n-\t\t\tsenddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01_w0);\n-\t\t\tsenddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23_w0);\n-\t\t\tsenddesc01_w1 = vorrq_u64(senddesc01_w1, xmask01_w1);\n-\t\t\tsenddesc23_w1 = vorrq_u64(senddesc23_w1, xmask23_w1);\n+\t\t\tcn10k_nix_prefree_seg_vec(tx_pkts, txq, &senddesc01_w0, &senddesc23_w0,\n+\t\t\t\t\t\t  &senddesc01_w1, &senddesc23_w1);\n \t\t} else if (!(flags & NIX_TX_MULTI_SEG_F) &&\n \t\t\t   !(flags & NIX_TX_OFFLOAD_SECURITY_F)) {\n \t\t\t/* Move mbufs to iova */\ndiff --git a/drivers/net/cnxk/cn9k_ethdev.c b/drivers/net/cnxk/cn9k_ethdev.c\nindex 67f21a9c7f..ea92b1dcb6 100644\n--- a/drivers/net/cnxk/cn9k_ethdev.c\n+++ b/drivers/net/cnxk/cn9k_ethdev.c\n@@ -347,7 +347,13 @@ cn9k_nix_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t qidx)\n \t\tstruct roc_nix_sq *sq = &dev->sqs[qidx];\n \t\tdo {\n \t\t\thandle_tx_completion_pkts(txq, 0);\n+\t\t\t/* Check if SQ is empty */\n \t\t\troc_nix_sq_head_tail_get(nix, sq->qid, &head, &tail);\n+\t\t\tif (head != tail)\n+\t\t\t\tcontinue;\n+\n+\t\t\t/* Check if completion CQ is empty */\n+\t\t\troc_nix_cq_head_tail_get(nix, sq->cqid, &head, &tail);\n \t\t} while (head != tail);\n \t}\n \ndiff --git a/drivers/net/cnxk/cn9k_ethdev.h b/drivers/net/cnxk/cn9k_ethdev.h\nindex 9e0a3c5bb2..6ae0db62ca 100644\n--- a/drivers/net/cnxk/cn9k_ethdev.h\n+++ b/drivers/net/cnxk/cn9k_ethdev.h\n@@ -169,6 +169,7 @@ handle_tx_completion_pkts(struct cn9k_eth_txq *txq, uint8_t mt_safe)\n \t\t\tm = m_next;\n \t\t}\n \t\trte_pktmbuf_free_seg(m);\n+\t\ttxq->tx_compl.ptr[tx_compl_s0->sqe_id] = NULL;\n \n \t\thead++;\n \t\thead &= qmask;\ndiff --git a/drivers/net/cnxk/cn9k_tx.h b/drivers/net/cnxk/cn9k_tx.h\nindex fba4bb4215..f28cecebd0 100644\n--- a/drivers/net/cnxk/cn9k_tx.h\n+++ b/drivers/net/cnxk/cn9k_tx.h\n@@ -83,9 +83,10 @@ cn9k_nix_tx_skeleton(struct cn9k_eth_txq *txq, uint64_t *cmd,\n }\n \n static __rte_always_inline uint64_t\n-cn9k_nix_prefree_seg(struct rte_mbuf *m, struct cn9k_eth_txq *txq,\n-\t\tstruct nix_send_hdr_s *send_hdr)\n+cn9k_nix_prefree_seg(struct rte_mbuf *m, struct cn9k_eth_txq *txq, struct nix_send_hdr_s *send_hdr,\n+\t\t     uint64_t *aura)\n {\n+\tstruct rte_mbuf *prev;\n \tuint32_t sqe_id;\n \n \tif (RTE_MBUF_HAS_EXTBUF(m)) {\n@@ -94,7 +95,10 @@ cn9k_nix_prefree_seg(struct rte_mbuf *m, struct cn9k_eth_txq *txq,\n \t\t\treturn 1;\n \t\t}\n \t\tif (send_hdr->w0.pnc) {\n-\t\t\ttxq->tx_compl.ptr[send_hdr->w1.sqe_id]->next = m;\n+\t\t\tsqe_id = send_hdr->w1.sqe_id;\n+\t\t\tprev = txq->tx_compl.ptr[sqe_id];\n+\t\t\tm->next = prev;\n+\t\t\ttxq->tx_compl.ptr[sqe_id] = m;\n \t\t} else {\n \t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n \t\t\tsend_hdr->w0.pnc = 1;\n@@ -104,10 +108,151 @@ cn9k_nix_prefree_seg(struct rte_mbuf *m, struct cn9k_eth_txq *txq,\n \t\t}\n \t\treturn 1;\n \t} else {\n-\t\treturn cnxk_nix_prefree_seg(m);\n+\t\treturn cnxk_nix_prefree_seg(m, aura);\n \t}\n }\n \n+#if defined(RTE_ARCH_ARM64)\n+/* Only called for first segments of single segmented mbufs */\n+static __rte_always_inline void\n+cn9k_nix_prefree_seg_vec(struct rte_mbuf **mbufs, struct cn9k_eth_txq *txq,\n+\t\t\t uint64x2_t *senddesc01_w0, uint64x2_t *senddesc23_w0,\n+\t\t\t uint64x2_t *senddesc01_w1, uint64x2_t *senddesc23_w1)\n+{\n+\tstruct rte_mbuf **tx_compl_ptr = txq->tx_compl.ptr;\n+\tuint32_t nb_desc_mask = txq->tx_compl.nb_desc_mask;\n+\tbool tx_compl_ena = txq->tx_compl.ena;\n+\tstruct rte_mbuf *m0, *m1, *m2, *m3;\n+\tstruct rte_mbuf *cookie;\n+\tuint64_t w0, w1, aura;\n+\tuint64_t sqe_id;\n+\n+\tm0 = mbufs[0];\n+\tm1 = mbufs[1];\n+\tm2 = mbufs[2];\n+\tm3 = mbufs[3];\n+\n+\t/* mbuf 0 */\n+\tw0 = vgetq_lane_u64(*senddesc01_w0, 0);\n+\tif (RTE_MBUF_HAS_EXTBUF(m0)) {\n+\t\tw0 |= BIT_ULL(19);\n+\t\tw1 = vgetq_lane_u64(*senddesc01_w1, 0);\n+\t\tw1 &= ~0xFFFF000000000000UL;\n+\t\tif (unlikely(!tx_compl_ena)) {\n+\t\t\trte_pktmbuf_free_seg(m0);\n+\t\t} else {\n+\t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n+\t\t\tsqe_id = sqe_id & nb_desc_mask;\n+\t\t\t/* Set PNC */\n+\t\t\tw0 |= BIT_ULL(43);\n+\t\t\tw1 |= sqe_id << 48;\n+\t\t\ttx_compl_ptr[sqe_id] = m0;\n+\t\t\t*senddesc01_w1 = vsetq_lane_u64(w1, *senddesc01_w1, 0);\n+\t\t}\n+\t} else {\n+\t\tcookie = RTE_MBUF_DIRECT(m0) ? m0 : rte_mbuf_from_indirect(m0);\n+\t\taura = (w0 >> 20) & 0xFFFFF;\n+\t\tw0 &= ~0xFFFFF00000UL;\n+\t\tw0 |= cnxk_nix_prefree_seg(m0, &aura) << 19;\n+\t\tw0 |= aura << 20;\n+\n+\t\tif ((w0 & BIT_ULL(19)) == 0)\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, &cookie, 1, 0);\n+\t}\n+\t*senddesc01_w0 = vsetq_lane_u64(w0, *senddesc01_w0, 0);\n+\n+\t/* mbuf1 */\n+\tw0 = vgetq_lane_u64(*senddesc01_w0, 1);\n+\tif (RTE_MBUF_HAS_EXTBUF(m1)) {\n+\t\tw0 |= BIT_ULL(19);\n+\t\tw1 = vgetq_lane_u64(*senddesc01_w1, 1);\n+\t\tw1 &= ~0xFFFF000000000000UL;\n+\t\tif (unlikely(!tx_compl_ena)) {\n+\t\t\trte_pktmbuf_free_seg(m1);\n+\t\t} else {\n+\t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n+\t\t\tsqe_id = sqe_id & nb_desc_mask;\n+\t\t\t/* Set PNC */\n+\t\t\tw0 |= BIT_ULL(43);\n+\t\t\tw1 |= sqe_id << 48;\n+\t\t\ttx_compl_ptr[sqe_id] = m1;\n+\t\t\t*senddesc01_w1 = vsetq_lane_u64(w1, *senddesc01_w1, 1);\n+\t\t}\n+\t} else {\n+\t\tcookie = RTE_MBUF_DIRECT(m1) ? m1 : rte_mbuf_from_indirect(m1);\n+\t\taura = (w0 >> 20) & 0xFFFFF;\n+\t\tw0 &= ~0xFFFFF00000UL;\n+\t\tw0 |= cnxk_nix_prefree_seg(m1, &aura) << 19;\n+\t\tw0 |= aura << 20;\n+\n+\t\tif ((w0 & BIT_ULL(19)) == 0)\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, &cookie, 1, 0);\n+\t}\n+\t*senddesc01_w0 = vsetq_lane_u64(w0, *senddesc01_w0, 1);\n+\n+\t/* mbuf 2 */\n+\tw0 = vgetq_lane_u64(*senddesc23_w0, 0);\n+\tif (RTE_MBUF_HAS_EXTBUF(m2)) {\n+\t\tw0 |= BIT_ULL(19);\n+\t\tw1 = vgetq_lane_u64(*senddesc23_w1, 0);\n+\t\tw1 &= ~0xFFFF000000000000UL;\n+\t\tif (unlikely(!tx_compl_ena)) {\n+\t\t\trte_pktmbuf_free_seg(m2);\n+\t\t} else {\n+\t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n+\t\t\tsqe_id = sqe_id & nb_desc_mask;\n+\t\t\t/* Set PNC */\n+\t\t\tw0 |= BIT_ULL(43);\n+\t\t\tw1 |= sqe_id << 48;\n+\t\t\ttx_compl_ptr[sqe_id] = m2;\n+\t\t\t*senddesc23_w1 = vsetq_lane_u64(w1, *senddesc23_w1, 0);\n+\t\t}\n+\t} else {\n+\t\tcookie = RTE_MBUF_DIRECT(m2) ? m2 : rte_mbuf_from_indirect(m2);\n+\t\taura = (w0 >> 20) & 0xFFFFF;\n+\t\tw0 &= ~0xFFFFF00000UL;\n+\t\tw0 |= cnxk_nix_prefree_seg(m2, &aura) << 19;\n+\t\tw0 |= aura << 20;\n+\n+\t\tif ((w0 & BIT_ULL(19)) == 0)\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, &cookie, 1, 0);\n+\t}\n+\t*senddesc23_w0 = vsetq_lane_u64(w0, *senddesc23_w0, 0);\n+\n+\t/* mbuf3 */\n+\tw0 = vgetq_lane_u64(*senddesc23_w0, 1);\n+\tif (RTE_MBUF_HAS_EXTBUF(m3)) {\n+\t\tw0 |= BIT_ULL(19);\n+\t\tw1 = vgetq_lane_u64(*senddesc23_w1, 1);\n+\t\tw1 &= ~0xFFFF000000000000UL;\n+\t\tif (unlikely(!tx_compl_ena)) {\n+\t\t\trte_pktmbuf_free_seg(m3);\n+\t\t} else {\n+\t\t\tsqe_id = __atomic_fetch_add(&txq->tx_compl.sqe_id, 1, __ATOMIC_RELAXED);\n+\t\t\tsqe_id = sqe_id & nb_desc_mask;\n+\t\t\t/* Set PNC */\n+\t\t\tw0 |= BIT_ULL(43);\n+\t\t\tw1 |= sqe_id << 48;\n+\t\t\ttx_compl_ptr[sqe_id] = m3;\n+\t\t\t*senddesc23_w1 = vsetq_lane_u64(w1, *senddesc23_w1, 1);\n+\t\t}\n+\t} else {\n+\t\tcookie = RTE_MBUF_DIRECT(m3) ? m3 : rte_mbuf_from_indirect(m3);\n+\t\taura = (w0 >> 20) & 0xFFFFF;\n+\t\tw0 &= ~0xFFFFF00000UL;\n+\t\tw0 |= cnxk_nix_prefree_seg(m3, &aura) << 19;\n+\t\tw0 |= aura << 20;\n+\n+\t\tif ((w0 & BIT_ULL(19)) == 0)\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, &cookie, 1, 0);\n+\t}\n+\t*senddesc23_w0 = vsetq_lane_u64(w0, *senddesc23_w0, 1);\n+#ifndef RTE_LIBRTE_MEMPOOL_DEBUG\n+\tRTE_SET_USED(cookie);\n+#endif\n+}\n+#endif\n+\n static __rte_always_inline void\n cn9k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags)\n {\n@@ -191,6 +336,8 @@ cn9k_nix_xmit_prepare(struct cn9k_eth_txq *txq,\n \t\tol_flags = m->ol_flags;\n \t\tw1.u = 0;\n \t}\n+\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)\n+\t\tsend_hdr->w0.pnc = 0;\n \n \tif (!(flags & NIX_TX_MULTI_SEG_F))\n \t\tsend_hdr->w0.total = m->data_len;\n@@ -345,23 +492,33 @@ cn9k_nix_xmit_prepare(struct cn9k_eth_txq *txq,\n \t\tsend_hdr->w1.u = w1.u;\n \n \tif (!(flags & NIX_TX_MULTI_SEG_F)) {\n+\t\tstruct rte_mbuf *cookie;\n+\n \t\tsg->seg1_size = m->data_len;\n \t\t*(rte_iova_t *)(++sg) = rte_mbuf_data_iova(m);\n+\t\tcookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m);\n \n \t\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {\n+\t\t\tuint64_t aura;\n \t\t\t/* DF bit = 1 if refcount of current mbuf or parent mbuf\n \t\t\t *\t\tis greater than 1\n \t\t\t * DF bit = 0 otherwise\n \t\t\t */\n-\t\t\tsend_hdr->w0.df = cn9k_nix_prefree_seg(m, txq, send_hdr);\n+\t\t\taura = send_hdr->w0.aura;\n+\t\t\tsend_hdr->w0.df = cn9k_nix_prefree_seg(m, txq, send_hdr, &aura);\n+\t\t\tsend_hdr->w0.aura = aura;\n \t\t\t/* Ensuring mbuf fields which got updated in\n \t\t\t * cnxk_nix_prefree_seg are written before LMTST.\n \t\t\t */\n \t\t\trte_io_wmb();\n \t\t}\n+#ifdef RTE_LIBRTE_MEMPOOL_DEBUG\n \t\t/* Mark mempool object as \"put\" since it is freed by NIX */\n \t\tif (!send_hdr->w0.df)\n \t\t\tRTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);\n+#else\n+\t\tRTE_SET_USED(cookie);\n+#endif\n \t} else {\n \t\tsg->seg1_size = m->data_len;\n \t\t*(rte_iova_t *)(sg + 1) = rte_mbuf_data_iova(m);\n@@ -443,6 +600,8 @@ cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq,\n \t\t      struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)\n {\n \tstruct nix_send_hdr_s *send_hdr;\n+\tuint64_t prefree = 0, aura;\n+\tstruct rte_mbuf *cookie;\n \tunion nix_send_sg_s *sg;\n \tstruct rte_mbuf *m_next;\n \tuint64_t *slist, sg_u;\n@@ -467,9 +626,13 @@ cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq,\n \tm_next = m->next;\n \tslist = &cmd[3 + off + 1];\n \n+\tcookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m);\n \t/* Set invert df if buffer is not to be freed by H/W */\n \tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {\n-\t\tsg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr) << 55);\n+\t\taura = send_hdr->w0.aura;\n+\t\tprefree = (cn9k_nix_prefree_seg(m, txq, send_hdr, &aura) << 55);\n+\t\tsend_hdr->w0.aura = aura;\n+\t\tsg_u |= prefree;\n \t\trte_io_wmb();\n \t}\n \n@@ -478,6 +641,8 @@ cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq,\n \tif (!(sg_u & (1ULL << 55)))\n \t\tRTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);\n \trte_io_wmb();\n+#else\n+\tRTE_SET_USED(cookie);\n #endif\n \tm = m_next;\n \tif (!m)\n@@ -490,7 +655,7 @@ cn9k_nix_prepare_mseg(struct cn9k_eth_txq *txq,\n \t\t*slist = rte_mbuf_data_iova(m);\n \t\t/* Set invert df if buffer is not to be freed by H/W */\n \t\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {\n-\t\t\tsg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr) << (i + 55));\n+\t\t\tsg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr, NULL) << (i + 55));\n \t\t\t/* Commit changes to mbuf */\n \t\t\trte_io_wmb();\n \t\t}\n@@ -709,8 +874,8 @@ cn9k_nix_prepare_mseg_vec_list(struct cn9k_eth_txq *txq,\n \t\t\t       struct nix_send_hdr_s *send_hdr,\n \t\t\t       union nix_send_sg_s *sg, const uint32_t flags)\n {\n-\tstruct rte_mbuf *m_next;\n-\tuint64_t *slist, sg_u;\n+\tstruct rte_mbuf *m_next, *cookie;\n+\tuint64_t *slist, sg_u, aura;\n \tuint16_t nb_segs;\n \tuint64_t segdw;\n \tint i = 1;\n@@ -727,13 +892,19 @@ cn9k_nix_prepare_mseg_vec_list(struct cn9k_eth_txq *txq,\n \tm_next = m->next;\n \n \t/* Set invert df if buffer is not to be freed by H/W */\n-\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)\n-\t\tsg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr) << 55);\n-\t\t/* Mark mempool object as \"put\" since it is freed by NIX */\n+\tcookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m);\n+\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {\n+\t\taura = send_hdr->w0.aura;\n+\t\tsg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr, &aura) << 55);\n+\t\tsend_hdr->w0.aura = aura;\n+\t}\n+\t/* Mark mempool object as \"put\" since it is freed by NIX */\n #ifdef RTE_LIBRTE_MEMPOOL_DEBUG\n \tif (!(sg_u & (1ULL << 55)))\n-\t\tRTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);\n+\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0);\n \trte_io_wmb();\n+#else\n+\tRTE_SET_USED(cookie);\n #endif\n \n \tm = m_next;\n@@ -742,14 +913,15 @@ cn9k_nix_prepare_mseg_vec_list(struct cn9k_eth_txq *txq,\n \t\tm_next = m->next;\n \t\tsg_u = sg_u | ((uint64_t)m->data_len << (i << 4));\n \t\t*slist = rte_mbuf_data_iova(m);\n+\t\tcookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m);\n \t\t/* Set invert df if buffer is not to be freed by H/W */\n \t\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)\n-\t\t\tsg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr) << (i + 55));\n+\t\t\tsg_u |= (cn9k_nix_prefree_seg(m, txq, send_hdr, &aura) << (i + 55));\n \t\t\t/* Mark mempool object as \"put\" since it is freed by NIX\n \t\t\t */\n #ifdef RTE_LIBRTE_MEMPOOL_DEBUG\n \t\tif (!(sg_u & (1ULL << (i + 55))))\n-\t\t\tRTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0);\n \t\trte_io_wmb();\n #endif\n \t\tslist++;\n@@ -789,15 +961,20 @@ cn9k_nix_prepare_mseg_vec(struct cn9k_eth_txq *txq,\n \t\t\t  uint64x2_t *cmd1, const uint32_t flags)\n {\n \tstruct nix_send_hdr_s send_hdr;\n+\tstruct rte_mbuf *cookie;\n \tunion nix_send_sg_s sg;\n+\tuint64_t aura;\n \tuint8_t ret;\n \n \tif (m->nb_segs == 1) {\n+\t\tcookie = RTE_MBUF_DIRECT(m) ? m : rte_mbuf_from_indirect(m);\n \t\tif (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {\n \t\t\tsend_hdr.w0.u = vgetq_lane_u64(cmd0[0], 0);\n \t\t\tsend_hdr.w1.u = vgetq_lane_u64(cmd0[0], 1);\n \t\t\tsg.u = vgetq_lane_u64(cmd1[0], 0);\n-\t\t\tsg.u |= (cn9k_nix_prefree_seg(m, txq, &send_hdr) << 55);\n+\t\t\taura = send_hdr.w0.aura;\n+\t\t\tsg.u |= (cn9k_nix_prefree_seg(m, txq, &send_hdr, &aura) << 55);\n+\t\t\tsend_hdr.w0.aura = aura;\n \t\t\tcmd1[0] = vsetq_lane_u64(sg.u, cmd1[0], 0);\n \t\t\tcmd0[0] = vsetq_lane_u64(send_hdr.w0.u, cmd0[0], 0);\n \t\t\tcmd0[0] = vsetq_lane_u64(send_hdr.w1.u, cmd0[0], 1);\n@@ -806,8 +983,10 @@ cn9k_nix_prepare_mseg_vec(struct cn9k_eth_txq *txq,\n #ifdef RTE_LIBRTE_MEMPOOL_DEBUG\n \t\tsg.u = vgetq_lane_u64(cmd1[0], 0);\n \t\tif (!(sg.u & (1ULL << 55)))\n-\t\t\tRTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);\n+\t\t\tRTE_MEMPOOL_CHECK_COOKIES(cookie->pool, (void **)&cookie, 1, 0);\n \t\trte_io_wmb();\n+#else\n+\t\tRTE_SET_USED(cookie);\n #endif\n \t\treturn 2 + !!(flags & NIX_TX_NEED_EXT_HDR) +\n \t\t       !!(flags & NIX_TX_OFFLOAD_TSTAMP_F);\n@@ -962,10 +1141,7 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,\n \tuint64x2_t sgdesc01_w1, sgdesc23_w1;\n \tstruct cn9k_eth_txq *txq = tx_queue;\n \tuint64_t *lmt_addr = txq->lmt_addr;\n-\tuint64x2_t xmask01_w0, xmask23_w0;\n-\tuint64x2_t xmask01_w1, xmask23_w1;\n \trte_iova_t io_addr = txq->io_addr;\n-\tstruct nix_send_hdr_s send_hdr;\n \tuint64x2_t ltypes01, ltypes23;\n \tuint64x2_t xtmp128, ytmp128;\n \tuint64x2_t xmask01, xmask23;\n@@ -1028,7 +1204,7 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,\n \tfor (i = 0; i < pkts; i += NIX_DESCS_PER_LOOP) {\n \t\t/* Clear lower 32bit of SEND_HDR_W0 and SEND_SG_W0 */\n \t\tsenddesc01_w0 =\n-\t\t\tvbicq_u64(senddesc01_w0, vdupq_n_u64(0xFFFFFFFF));\n+\t\t\tvbicq_u64(senddesc01_w0, vdupq_n_u64(0x800FFFFFFFF));\n \t\tsgdesc01_w0 = vbicq_u64(sgdesc01_w0, vdupq_n_u64(0xFFFFFFFF));\n \n \t\tsenddesc23_w0 = senddesc01_w0;\n@@ -1732,74 +1908,8 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,\n \t\tif ((flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) &&\n \t\t    !(flags & NIX_TX_MULTI_SEG_F)) {\n \t\t\t/* Set don't free bit if reference count > 1 */\n-\t\t\txmask01_w0 = vdupq_n_u64(0);\n-\t\t\txmask01_w1 = vdupq_n_u64(0);\n-\t\t\txmask23_w0 = xmask01_w0;\n-\t\t\txmask23_w1 = xmask01_w1;\n-\n-\t\t\t/* Move mbufs to iova */\n-\t\t\tmbuf0 = (uint64_t *)tx_pkts[0];\n-\t\t\tmbuf1 = (uint64_t *)tx_pkts[1];\n-\t\t\tmbuf2 = (uint64_t *)tx_pkts[2];\n-\t\t\tmbuf3 = (uint64_t *)tx_pkts[3];\n-\n-\t\t\tsend_hdr.w0.u = 0;\n-\t\t\tsend_hdr.w1.u = 0;\n-\n-\t\t\tif (cn9k_nix_prefree_seg((struct rte_mbuf *)mbuf0, txq, &send_hdr)) {\n-\t\t\t\tsend_hdr.w0.df = 1;\n-\t\t\t\txmask01_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask01_w0, 0);\n-\t\t\t\txmask01_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask01_w1, 0);\n-\t\t\t} else {\n-\t\t\t\tRTE_MEMPOOL_CHECK_COOKIES(\n-\t\t\t\t\t((struct rte_mbuf *)mbuf0)->pool,\n-\t\t\t\t\t(void **)&mbuf0, 1, 0);\n-\t\t\t}\n-\n-\t\t\tsend_hdr.w0.u = 0;\n-\t\t\tsend_hdr.w1.u = 0;\n-\n-\t\t\tif (cn9k_nix_prefree_seg((struct rte_mbuf *)mbuf1, txq, &send_hdr)) {\n-\t\t\t\tsend_hdr.w0.df = 1;\n-\t\t\t\txmask01_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask01_w0, 1);\n-\t\t\t\txmask01_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask01_w1, 1);\n-\t\t\t} else {\n-\t\t\t\tRTE_MEMPOOL_CHECK_COOKIES(\n-\t\t\t\t\t((struct rte_mbuf *)mbuf1)->pool,\n-\t\t\t\t\t(void **)&mbuf1, 1, 0);\n-\t\t\t}\n-\n-\t\t\tsend_hdr.w0.u = 0;\n-\t\t\tsend_hdr.w1.u = 0;\n-\n-\t\t\tif (cn9k_nix_prefree_seg((struct rte_mbuf *)mbuf2, txq, &send_hdr)) {\n-\t\t\t\tsend_hdr.w0.df = 1;\n-\t\t\t\txmask23_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask23_w0, 0);\n-\t\t\t\txmask23_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask23_w1, 0);\n-\t\t\t} else {\n-\t\t\t\tRTE_MEMPOOL_CHECK_COOKIES(\n-\t\t\t\t\t((struct rte_mbuf *)mbuf2)->pool,\n-\t\t\t\t\t(void **)&mbuf2, 1, 0);\n-\t\t\t}\n-\n-\t\t\tsend_hdr.w0.u = 0;\n-\t\t\tsend_hdr.w1.u = 0;\n-\n-\t\t\tif (cn9k_nix_prefree_seg((struct rte_mbuf *)mbuf3, txq, &send_hdr)) {\n-\t\t\t\tsend_hdr.w0.df = 1;\n-\t\t\t\txmask23_w0 = vsetq_lane_u64(send_hdr.w0.u, xmask23_w0, 1);\n-\t\t\t\txmask23_w1 = vsetq_lane_u64(send_hdr.w1.u, xmask23_w1, 1);\n-\t\t\t} else {\n-\t\t\t\tRTE_MEMPOOL_CHECK_COOKIES(\n-\t\t\t\t\t((struct rte_mbuf *)mbuf3)->pool,\n-\t\t\t\t\t(void **)&mbuf3, 1, 0);\n-\t\t\t}\n-\n-\t\t\tsenddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01_w0);\n-\t\t\tsenddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23_w0);\n-\t\t\tsenddesc01_w1 = vorrq_u64(senddesc01_w1, xmask01_w1);\n-\t\t\tsenddesc23_w1 = vorrq_u64(senddesc23_w1, xmask23_w1);\n-\n+\t\t\tcn9k_nix_prefree_seg_vec(tx_pkts, txq, &senddesc01_w0, &senddesc23_w0,\n+\t\t\t\t\t\t &senddesc01_w1, &senddesc23_w1);\n \t\t\t/* Ensuring mbuf fields which got updated in\n \t\t\t * cnxk_nix_prefree_seg are written before LMTST.\n \t\t\t */\ndiff --git a/drivers/net/cnxk/cnxk_ethdev_dp.h b/drivers/net/cnxk/cnxk_ethdev_dp.h\nindex c1f99a2616..67f40b8e25 100644\n--- a/drivers/net/cnxk/cnxk_ethdev_dp.h\n+++ b/drivers/net/cnxk/cnxk_ethdev_dp.h\n@@ -84,7 +84,7 @@ struct cnxk_timesync_info {\n \n /* Inlines */\n static __rte_always_inline uint64_t\n-cnxk_pktmbuf_detach(struct rte_mbuf *m)\n+cnxk_pktmbuf_detach(struct rte_mbuf *m, uint64_t *aura)\n {\n \tstruct rte_mempool *mp = m->pool;\n \tuint32_t mbuf_size, buf_len;\n@@ -94,6 +94,8 @@ cnxk_pktmbuf_detach(struct rte_mbuf *m)\n \n \t/* Update refcount of direct mbuf */\n \tmd = rte_mbuf_from_indirect(m);\n+\tif (aura)\n+\t\t*aura = roc_npa_aura_handle_to_aura(md->pool->pool_id);\n \trefcount = rte_mbuf_refcnt_update(md, -1);\n \n \tpriv_size = rte_pktmbuf_priv_size(mp);\n@@ -126,18 +128,18 @@ cnxk_pktmbuf_detach(struct rte_mbuf *m)\n }\n \n static __rte_always_inline uint64_t\n-cnxk_nix_prefree_seg(struct rte_mbuf *m)\n+cnxk_nix_prefree_seg(struct rte_mbuf *m, uint64_t *aura)\n {\n \tif (likely(rte_mbuf_refcnt_read(m) == 1)) {\n \t\tif (!RTE_MBUF_DIRECT(m))\n-\t\t\treturn cnxk_pktmbuf_detach(m);\n+\t\t\treturn cnxk_pktmbuf_detach(m, aura);\n \n \t\tm->next = NULL;\n \t\tm->nb_segs = 1;\n \t\treturn 0;\n \t} else if (rte_mbuf_refcnt_update(m, -1) == 0) {\n \t\tif (!RTE_MBUF_DIRECT(m))\n-\t\t\treturn cnxk_pktmbuf_detach(m);\n+\t\t\treturn cnxk_pktmbuf_detach(m, aura);\n \n \t\trte_mbuf_refcnt_set(m, 1);\n \t\tm->next = NULL;\n",
    "prefixes": [
        "10/13"
    ]
}