Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/121083/?format=api
{ "id": 121083, "url": "http://patches.dpdk.org/api/patches/121083/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20221220143232.2519650-10-ktejasree@marvell.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20221220143232.2519650-10-ktejasree@marvell.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20221220143232.2519650-10-ktejasree@marvell.com", "date": "2022-12-20T14:32:24", "name": "[09/17] crypto/cnxk: add CN9K IPsec SG support", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "b20a89ad4506c1cded2bb9976632e24684507bc4", "submitter": { "id": 1789, "url": "http://patches.dpdk.org/api/people/1789/?format=api", "name": "Tejasree Kondoj", "email": "ktejasree@marvell.com" }, "delegate": { "id": 6690, "url": "http://patches.dpdk.org/api/users/6690/?format=api", "username": "akhil", "first_name": "akhil", "last_name": "goyal", "email": "gakhil@marvell.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20221220143232.2519650-10-ktejasree@marvell.com/mbox/", "series": [ { "id": 26195, "url": "http://patches.dpdk.org/api/series/26195/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=26195", "date": "2022-12-20T14:32:15", "name": "fixes and improvements to cnxk crytpo PMD", "version": 1, "mbox": "http://patches.dpdk.org/series/26195/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/121083/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/121083/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 43D10A0545;\n\tTue, 20 Dec 2022 15:33:46 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 720DA410DE;\n\tTue, 20 Dec 2022 15:33:07 +0100 (CET)", "from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com\n [67.231.156.173])\n by mails.dpdk.org (Postfix) with ESMTP id 3B05D42D40\n for <dev@dpdk.org>; Tue, 20 Dec 2022 15:33:05 +0100 (CET)", "from pps.filterd (m0045851.ppops.net [127.0.0.1])\n by mx0b-0016f401.pphosted.com (8.17.1.19/8.17.1.19) with ESMTP id\n 2BKEJKMV010037 for <dev@dpdk.org>; Tue, 20 Dec 2022 06:33:04 -0800", "from dc5-exch02.marvell.com ([199.233.59.182])\n by mx0b-0016f401.pphosted.com (PPS) with ESMTPS id 3mhe5rnb5k-1\n (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT)\n for <dev@dpdk.org>; Tue, 20 Dec 2022 06:33:03 -0800", "from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH02.marvell.com\n (10.69.176.39) with Microsoft SMTP Server (TLS) id 15.0.1497.42;\n Tue, 20 Dec 2022 06:33:01 -0800", "from maili.marvell.com (10.69.176.80) by DC5-EXCH02.marvell.com\n (10.69.176.39) with Microsoft SMTP Server id 15.0.1497.42 via Frontend\n Transport; Tue, 20 Dec 2022 06:33:01 -0800", "from hyd1554.marvell.com (unknown [10.29.57.11])\n by maili.marvell.com (Postfix) with ESMTP id A0B0C3F704C;\n Tue, 20 Dec 2022 06:32:58 -0800 (PST)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com;\n h=from : to : cc :\n subject : date : message-id : in-reply-to : references : mime-version :\n content-transfer-encoding : content-type; s=pfpt0220;\n bh=2jlKhFo6ZMf6TGQybe0UjNwAYA79BsjUNwc02EOKR+8=;\n b=GOqhrK1r6Ot2LN8Z5VLVyn5Tv7ahpszDYt4ufRjIZ1FfSpJEBuGQOZEUVPddFy2WgDlZ\n mwytVGxh98givB2LOHmYAn/Lr4S1+vWo1k3Sw+LdmpQbulifGBdq9/3hkJpT9tQ4AuSJ\n BuUc6n69IDwPBqmKLZUyf3Bnm/8rIOtU1pLoMrqdWYRcrS0k3VmwFyhzuo5Arzp21279\n ISTbX8/CYtAxbsdRysJikzyDYWfBXdnivaFuWVmDUGYtxHc4YSuHk5I7uuCvXXw0a+36\n 0esdzksAD21vhdEx2VZA3Z0SEayLVlfgd/OymL2o/yYj4RHyrDHF8hK0aruuAqPDm4gN kg==", "From": "Tejasree Kondoj <ktejasree@marvell.com>", "To": "Akhil Goyal <gakhil@marvell.com>", "CC": "Archana Muniganti <marchana@marvell.com>, Anoob Joseph\n <anoobj@marvell.com>, Vidya Sagar Velumuri <vvelumuri@marvell.com>,\n Gowrishankar Muthukrishnan <gmuthukrishn@marvell.com>, Volodymyr Fialko\n <vfialko@marvell.com>,\n Aakash Sasidharan <asasidharan@marvell.com>, <dev@dpdk.org>", "Subject": "[PATCH 09/17] crypto/cnxk: add CN9K IPsec SG support", "Date": "Tue, 20 Dec 2022 20:02:24 +0530", "Message-ID": "<20221220143232.2519650-10-ktejasree@marvell.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20221220143232.2519650-1-ktejasree@marvell.com>", "References": "<20221220143232.2519650-1-ktejasree@marvell.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Content-Type": "text/plain", "X-Proofpoint-GUID": "nYsS0xngQnS6tZsRe6Vi0Kop6VKFtWqK", "X-Proofpoint-ORIG-GUID": "nYsS0xngQnS6tZsRe6Vi0Kop6VKFtWqK", "X-Proofpoint-Virus-Version": "vendor=baseguard\n engine=ICAP:2.0.205,Aquarius:18.0.923,Hydra:6.0.545,FMLib:17.11.122.1\n definitions=2022-12-20_05,2022-12-20_01,2022-06-22_01", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "From: Archana Muniganti <marchana@marvell.com>\n\nAdded IPsec SG support in CN9K\n\nSigned-off-by: Archana Muniganti <marchana@marvell.com>\nSigned-off-by: Tejasree Kondoj <ktejasree@marvell.com>\n---\n drivers/common/cnxk/roc_api.h | 1 +\n drivers/common/cnxk/roc_cpt_sg.h | 37 +++\n drivers/common/cnxk/roc_ie_on.h | 9 +-\n drivers/common/cnxk/roc_se.h | 28 --\n drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 75 ++---\n drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 171 ++++++++--\n drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 25 +-\n drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 19 ++\n drivers/crypto/cnxk/cnxk_se.h | 387 +++--------------------\n drivers/crypto/cnxk/cnxk_sg.h | 273 ++++++++++++++++\n 10 files changed, 594 insertions(+), 431 deletions(-)\n create mode 100644 drivers/common/cnxk/roc_cpt_sg.h\n create mode 100644 drivers/crypto/cnxk/cnxk_sg.h", "diff": "diff --git a/drivers/common/cnxk/roc_api.h b/drivers/common/cnxk/roc_api.h\nindex 072f16d77d..14a11321e0 100644\n--- a/drivers/common/cnxk/roc_api.h\n+++ b/drivers/common/cnxk/roc_api.h\n@@ -86,6 +86,7 @@\n /* CPT microcode */\n #include \"roc_ae.h\"\n #include \"roc_ae_fpm_tables.h\"\n+#include \"roc_cpt_sg.h\"\n #include \"roc_ie.h\"\n #include \"roc_ie_on.h\"\n #include \"roc_ie_ot.h\"\ndiff --git a/drivers/common/cnxk/roc_cpt_sg.h b/drivers/common/cnxk/roc_cpt_sg.h\nnew file mode 100644\nindex 0000000000..8a97e1aa5b\n--- /dev/null\n+++ b/drivers/common/cnxk/roc_cpt_sg.h\n@@ -0,0 +1,37 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(C) 2022 Marvell.\n+ */\n+\n+#ifndef _ROC_CPT_SG_H_\n+#define _ROC_CPT_SG_H_\n+\n+#define ROC_DMA_MODE_SG (1 << 7)\n+\n+#define ROC_MAX_SG_IN_OUT_CNT 32\n+#define ROC_MAX_SG_CNT\t (ROC_MAX_SG_IN_OUT_CNT / 2)\n+\n+#define ROC_SG_LIST_HDR_SIZE (8u)\n+#define ROC_SG_ENTRY_SIZE sizeof(struct roc_sglist_comp)\n+\n+struct roc_sglist_comp {\n+\tunion {\n+\t\tuint64_t len;\n+\t\tstruct {\n+\t\t\tuint16_t len[4];\n+\t\t} s;\n+\t} u;\n+\tuint64_t ptr[4];\n+};\n+\n+struct roc_sg2list_comp {\n+\tunion {\n+\t\tuint64_t len;\n+\t\tstruct {\n+\t\t\tuint16_t len[3];\n+\t\t\tuint16_t valid_segs;\n+\t\t} s;\n+\t} u;\n+\tuint64_t ptr[3];\n+};\n+\n+#endif /* _ROC_CPT_SG_H_ */\ndiff --git a/drivers/common/cnxk/roc_ie_on.h b/drivers/common/cnxk/roc_ie_on.h\nindex 057ff95362..9933ffa148 100644\n--- a/drivers/common/cnxk/roc_ie_on.h\n+++ b/drivers/common/cnxk/roc_ie_on.h\n@@ -25,10 +25,11 @@ enum roc_ie_on_ucc_ipsec {\n };\n \n /* Helper macros */\n-#define ROC_IE_ON_INB_RPTR_HDR 16\n-#define ROC_IE_ON_MAX_IV_LEN 16\n-#define ROC_IE_ON_PER_PKT_IV BIT(43)\n-#define ROC_IE_ON_INPLACE_BIT BIT(6)\n+#define ROC_IE_ON_OUTB_DPTR_HDR 16\n+#define ROC_IE_ON_INB_RPTR_HDR\t16\n+#define ROC_IE_ON_MAX_IV_LEN\t16\n+#define ROC_IE_ON_PER_PKT_IV\tBIT(43)\n+#define ROC_IE_ON_INPLACE_BIT\tBIT(6)\n \n enum {\n \tROC_IE_ON_SA_ENC_NULL = 0,\ndiff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h\nindex c357c19c0b..a8f0f49479 100644\n--- a/drivers/common/cnxk/roc_se.h\n+++ b/drivers/common/cnxk/roc_se.h\n@@ -27,13 +27,6 @@\n #define ROC_SE_MAX_MAC_LEN 64\n \n #define ROC_SE_OFF_CTRL_LEN 8\n-#define ROC_SE_DMA_MODE\t (1 << 7)\n-\n-#define ROC_SE_MAX_SG_IN_OUT_CNT 32\n-#define ROC_SE_MAX_SG_CNT\t (ROC_SE_MAX_SG_IN_OUT_CNT / 2)\n-\n-#define ROC_SE_SG_LIST_HDR_SIZE (8u)\n-#define ROC_SE_SG_ENTRY_SIZE\tsizeof(struct roc_se_sglist_comp)\n \n #define ROC_SE_ZS_EA 0x1\n #define ROC_SE_ZS_IA 0x2\n@@ -173,27 +166,6 @@ typedef enum {\n \tROC_SE_PDCP_MAC_LEN_128_BIT = 0x3\n } roc_se_pdcp_mac_len_type;\n \n-struct roc_se_sglist_comp {\n-\tunion {\n-\t\tuint64_t len;\n-\t\tstruct {\n-\t\t\tuint16_t len[4];\n-\t\t} s;\n-\t} u;\n-\tuint64_t ptr[4];\n-};\n-\n-struct roc_se_sg2list_comp {\n-\tunion {\n-\t\tuint64_t len;\n-\t\tstruct {\n-\t\t\tuint16_t len[3];\n-\t\t\tuint16_t valid_segs;\n-\t\t} s;\n-\t} u;\n-\tuint64_t ptr[3];\n-};\n-\n struct roc_se_enc_context {\n \tuint64_t iv_source : 1;\n \tuint64_t aes_key : 2;\ndiff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c\nindex 04c004bc7a..cfe1e08dff 100644\n--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c\n+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c\n@@ -18,13 +18,11 @@\n #include \"cnxk_se.h\"\n \n static __rte_always_inline int __rte_hot\n-cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,\n-\t\t struct cpt_inflight_req *infl_req,\n-\t\t struct cpt_inst_s *inst)\n+cn9k_cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,\n+\t\t struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst)\n {\n \tstruct rte_crypto_sym_op *sym_op = op->sym;\n \tstruct cn9k_sec_session *sec_sess;\n-\tint ret;\n \n \tsec_sess = (struct cn9k_sec_session *)(op->sym->session);\n \n@@ -33,22 +31,10 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,\n \t\treturn -ENOTSUP;\n \t}\n \n-\tif (unlikely(!rte_pktmbuf_is_contiguous(sym_op->m_src))) {\n-\t\tplt_dp_err(\"Scatter Gather mode is not supported\");\n-\t\treturn -ENOTSUP;\n-\t}\n-\n \tif (sec_sess->is_outbound)\n-\t\tret = process_outb_sa(op, sec_sess, inst);\n-\telse {\n-\t\tinfl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;\n-\t\tprocess_inb_sa(op, sec_sess, inst);\n-\t\tif (unlikely(sec_sess->replay_win_sz))\n-\t\t\tinfl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY;\n-\t\tret = 0;\n-\t}\n-\n-\treturn ret;\n+\t\treturn process_outb_sa(&qp->meta_info, op, sec_sess, inst, infl_req);\n+\telse\n+\t\treturn process_inb_sa(&qp->meta_info, op, sec_sess, inst, infl_req);\n }\n \n static inline struct cnxk_se_sess *\n@@ -94,7 +80,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,\n \t\t\tret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst, false);\n \t\t\tinst->w7.u64 = sess->cpt_inst_w7;\n \t\t} else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)\n-\t\t\tret = cn9k_cpt_sec_inst_fill(op, infl_req, inst);\n+\t\t\tret = cn9k_cpt_sec_inst_fill(qp, op, infl_req, inst);\n \t\telse {\n \t\t\tsess = cn9k_cpt_sym_temp_sess_create(qp, op);\n \t\t\tif (unlikely(sess == NULL)) {\n@@ -522,45 +508,60 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,\n {\n \tstruct rte_crypto_sym_op *sym_op = cop->sym;\n \tstruct rte_mbuf *m = sym_op->m_src;\n+\tstruct roc_ie_on_inb_hdr *hdr;\n \tstruct cn9k_sec_session *priv;\n \tstruct rte_ipv6_hdr *ip6;\n \tstruct rte_ipv4_hdr *ip;\n \tuint16_t m_len = 0;\n-\tchar *data;\n \n \tif (infl_req->op_flags & CPT_OP_FLAGS_IPSEC_DIR_INBOUND) {\n \n-\t\tdata = rte_pktmbuf_mtod(m, char *);\n-\t\tif (unlikely(infl_req->op_flags &\n-\t\t\t CPT_OP_FLAGS_IPSEC_INB_REPLAY)) {\n+\t\thdr = (struct roc_ie_on_inb_hdr *)rte_pktmbuf_mtod(m, char *);\n+\n+\t\tif (likely(m->next == NULL)) {\n+\t\t\tip = PLT_PTR_ADD(hdr, ROC_IE_ON_INB_RPTR_HDR);\n+\t\t} else {\n+\t\t\tip = (struct rte_ipv4_hdr *)hdr;\n+\t\t\thdr = infl_req->mdata;\n+\t\t}\n+\n+\t\tif (unlikely(infl_req->op_flags & CPT_OP_FLAGS_IPSEC_INB_REPLAY)) {\n \t\t\tint ret;\n \n \t\t\tpriv = (struct cn9k_sec_session *)(sym_op->session);\n \n-\t\t\tret = ipsec_antireplay_check(priv, priv->replay_win_sz,\n-\t\t\t\t\t\t (struct roc_ie_on_inb_hdr *)data);\n+\t\t\tret = ipsec_antireplay_check(priv, priv->replay_win_sz, hdr);\n \t\t\tif (unlikely(ret)) {\n \t\t\t\tcop->status = RTE_CRYPTO_OP_STATUS_ERROR;\n \t\t\t\treturn;\n \t\t\t}\n \t\t}\n \n-\t\tip = (struct rte_ipv4_hdr *)(data + ROC_IE_ON_INB_RPTR_HDR);\n-\n-\t\tif (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) ==\n-\t\t IPVERSION) {\n+\t\tif (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == IPVERSION) {\n \t\t\tm_len = rte_be_to_cpu_16(ip->total_length);\n \t\t} else {\n-\t\t\tPLT_ASSERT(((ip->version_ihl & 0xf0) >>\n-\t\t\t\t RTE_IPV4_IHL_MULTIPLIER) == 6);\n+\t\t\tPLT_ASSERT(((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == 6);\n \t\t\tip6 = (struct rte_ipv6_hdr *)ip;\n-\t\t\tm_len = rte_be_to_cpu_16(ip6->payload_len) +\n-\t\t\t\tsizeof(struct rte_ipv6_hdr);\n+\t\t\tm_len = rte_be_to_cpu_16(ip6->payload_len) + sizeof(struct rte_ipv6_hdr);\n \t\t}\n \n-\t\tm->data_len = m_len;\n-\t\tm->pkt_len = m_len;\n-\t\tm->data_off += ROC_IE_ON_INB_RPTR_HDR;\n+\t\tif (likely(m->next == NULL)) {\n+\t\t\tm->data_len = m_len;\n+\t\t\tm->pkt_len = m_len;\n+\n+\t\t\tm->data_off += ROC_IE_ON_INB_RPTR_HDR;\n+\t\t} else {\n+\t\t\tstruct rte_mbuf *temp = m;\n+\t\t\tuint8_t m_len_s = m_len;\n+\n+\t\t\twhile (m_len_s - temp->data_len > 0) {\n+\t\t\t\tm_len_s -= temp->data_len;\n+\t\t\t\ttemp = temp->next;\n+\t\t\t}\n+\n+\t\t\ttemp->data_len = m_len_s;\n+\t\t\tm->pkt_len = m_len;\n+\t\t}\n \t}\n }\n \ndiff --git a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h\nindex f1298017ce..3d9c851f10 100644\n--- a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h\n+++ b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h\n@@ -10,7 +10,9 @@\n #include <rte_security.h>\n \n #include \"cn9k_ipsec.h\"\n+#include \"cnxk_cryptodev_ops.h\"\n #include \"cnxk_security_ar.h\"\n+#include \"cnxk_sg.h\"\n \n static __rte_always_inline int32_t\n ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_mbuf *m_src)\n@@ -58,7 +60,9 @@ ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_m\n }\n \n static __rte_always_inline int\n-process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst)\n+process_outb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop,\n+\t\tstruct cn9k_sec_session *sess, struct cpt_inst_s *inst,\n+\t\tstruct cpt_inflight_req *infl_req)\n {\n \tconst unsigned int hdr_len = sess->custom_hdr_len;\n \tstruct rte_crypto_sym_op *sym_op = cop->sym;\n@@ -74,24 +78,90 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct\n \trlen = ipsec_po_out_rlen_get(sess, pkt_len, m_src);\n \n \textend_tail = rlen - dlen;\n-\tif (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {\n-\t\tplt_dp_err(\"Not enough tail room (required: %d, available: %d)\",\n-\t\t\t extend_tail, rte_pktmbuf_tailroom(m_src));\n-\t\treturn -ENOMEM;\n-\t}\n+\tpkt_len += extend_tail;\n \n-\tif (unlikely(hdr_len > data_off)) {\n-\t\tplt_dp_err(\"Not enough head room (required: %d, available: %d)\",\n-\t\t\t hdr_len, rte_pktmbuf_headroom(m_src));\n-\t\treturn -ENOMEM;\n-\t}\n+\tif (likely(m_src->next == NULL)) {\n+\t\tif (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {\n+\t\t\tplt_dp_err(\"Not enough tail room (required: %d, available: %d)\",\n+\t\t\t\t extend_tail, rte_pktmbuf_tailroom(m_src));\n+\t\t\treturn -ENOMEM;\n+\t\t}\n \n-\tpkt_len += extend_tail;\n+\t\tif (unlikely(hdr_len > data_off)) {\n+\t\t\tplt_dp_err(\"Not enough head room (required: %d, available: %d)\", hdr_len,\n+\t\t\t\t rte_pktmbuf_headroom(m_src));\n+\t\t\treturn -ENOMEM;\n+\t\t}\n \n-\tm_src->data_len = pkt_len;\n-\tm_src->pkt_len = pkt_len;\n+\t\tm_src->data_len = pkt_len;\n+\n+\t\thdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);\n+\n+\t\tinst->dptr = PLT_U64_CAST(hdr);\n+\t\tinst->w4.u64 = sess->inst.w4 | dlen;\n+\t} else {\n+\t\tstruct roc_sglist_comp *scatter_comp, *gather_comp;\n+\t\tuint32_t g_size_bytes, s_size_bytes;\n+\t\tstruct rte_mbuf *last_seg;\n+\t\tuint8_t *in_buffer;\n+\t\tvoid *m_data;\n+\t\tint i;\n+\n+\t\tlast_seg = rte_pktmbuf_lastseg(m_src);\n+\n+\t\tif (unlikely(extend_tail > rte_pktmbuf_tailroom(last_seg))) {\n+\t\t\tplt_dp_err(\"Not enough tail room (required: %d, available: %d)\",\n+\t\t\t\t extend_tail, rte_pktmbuf_tailroom(last_seg));\n+\t\t\treturn -ENOMEM;\n+\t\t}\n+\n+\t\tm_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);\n+\t\tif (unlikely(m_data == NULL)) {\n+\t\t\tplt_dp_err(\"Error allocating meta buffer for request\");\n+\t\t\treturn -ENOMEM;\n+\t\t}\n+\n+\t\thdr = m_data;\n+\n+\t\tm_data = (uint8_t *)m_data + hdr_len;\n+\t\tin_buffer = m_data;\n+\n+\t\t((uint16_t *)in_buffer)[0] = 0;\n+\t\t((uint16_t *)in_buffer)[1] = 0;\n+\n+\t\t/*\n+\t\t * Input Gather List\n+\t\t */\n+\t\ti = 0;\n+\t\tgather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);\n+\n+\t\ti = fill_sg_comp(gather_comp, i, (uint64_t)hdr, hdr_len);\n+\t\ti = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);\n+\t\t((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);\n+\n+\t\tg_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n+\n+\t\t/*\n+\t\t * output Scatter List\n+\t\t */\n+\t\tlast_seg->data_len += extend_tail;\n+\n+\t\ti = 0;\n+\t\tscatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);\n+\n+\t\ti = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len);\n+\t\ti = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);\n+\t\t((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);\n \n-\thdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);\n+\t\ts_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n+\n+\t\tdlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;\n+\n+\t\tinst->dptr = (uint64_t)in_buffer;\n+\n+\t\tinst->w4.u64 = sess->inst.w4 | dlen;\n+\t\tinst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;\n+\t}\n \n #ifdef LA_IPSEC_DEBUG\n \tif (sess->inst.w4 & ROC_IE_ON_PER_PKT_IV) {\n@@ -101,6 +171,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct\n \t}\n #endif\n \n+\tm_src->pkt_len = pkt_len;\n \tesn = ++sess->esn;\n \n \t/* Set ESN seq hi */\n@@ -114,22 +185,80 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct\n \thdr->ip_id = seq_lo;\n \n \t/* Prepare CPT instruction */\n-\tinst->w4.u64 = sess->inst.w4 | dlen;\n-\tinst->dptr = PLT_U64_CAST(hdr);\n \tinst->w7.u64 = sess->inst.w7;\n \n \treturn 0;\n }\n \n-static __rte_always_inline void\n-process_inb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst)\n+static __rte_always_inline int\n+process_inb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst, struct cpt_inflight_req *infl_req)\n {\n+\tconst unsigned int hdr_len = ROC_IE_ON_INB_RPTR_HDR;\n \tstruct rte_crypto_sym_op *sym_op = cop->sym;\n \tstruct rte_mbuf *m_src = sym_op->m_src;\n+\tstruct roc_ie_on_inb_hdr *hdr;\n+\tuint32_t dlen;\n+\n+\tinfl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;\n+\tif (likely(m_src->next == NULL)) {\n+\t\tdlen = rte_pktmbuf_pkt_len(m_src);\n+\t\tinst->dptr = rte_pktmbuf_mtod(m_src, uint64_t);\n+\t\tinst->w4.u64 = sess->inst.w4 | dlen;\n+\t} else {\n+\t\tstruct roc_sglist_comp *scatter_comp, *gather_comp;\n+\t\tuint32_t g_size_bytes, s_size_bytes;\n+\t\tuint8_t *in_buffer;\n+\t\tvoid *m_data;\n+\t\tint i;\n+\n+\t\tm_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);\n+\t\tif (unlikely(m_data == NULL)) {\n+\t\t\tplt_dp_err(\"Error allocating meta buffer for request\");\n+\t\t\treturn -ENOMEM;\n+\t\t}\n+\n+\t\thdr = m_data;\n+\n+\t\tm_data = (uint8_t *)m_data + hdr_len;\n+\t\tin_buffer = m_data;\n+\n+\t\t((uint16_t *)in_buffer)[0] = 0;\n+\t\t((uint16_t *)in_buffer)[1] = 0;\n+\n+\t\t/*\n+\t\t * Input Gather List\n+\t\t */\n+\t\ti = 0;\n+\t\tgather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);\n+\t\ti = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);\n+\t\t((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);\n+\n+\t\tg_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n+\n+\t\t/*\n+\t\t * Output Scatter List\n+\t\t */\n+\t\ti = 0;\n+\t\tscatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);\n+\t\ti = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len);\n+\t\ti = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);\n+\t\t((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);\n+\n+\t\ts_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n+\n+\t\tdlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;\n+\n+\t\tinst->dptr = (uint64_t)in_buffer;\n+\t\tinst->w4.u64 = sess->inst.w4 | dlen;\n+\t\tinst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;\n+\t}\n \n \t/* Prepare CPT instruction */\n-\tinst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);\n-\tinst->dptr = rte_pktmbuf_mtod(m_src, uint64_t);\n \tinst->w7.u64 = sess->inst.w7;\n+\n+\tif (unlikely(sess->replay_win_sz))\n+\t\tinfl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY;\n+\n+\treturn 0;\n }\n #endif /* __CN9K_IPSEC_LA_OPS_H__ */\ndiff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c\nindex a9c42205e6..eb2ed0d103 100644\n--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c\n+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c\n@@ -30,10 +30,22 @@ cnxk_cpt_get_mlen(void)\n \t/* For PDCP_CHAIN passthrough alignment */\n \tlen += 8;\n \tlen += ROC_SE_OFF_CTRL_LEN + ROC_CPT_AES_CBC_IV_LEN;\n-\tlen += RTE_ALIGN_CEIL(\n-\t\t(ROC_SE_SG_LIST_HDR_SIZE +\n-\t\t (RTE_ALIGN_CEIL(ROC_SE_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SE_SG_ENTRY_SIZE),\n-\t\t8);\n+\tlen += RTE_ALIGN_CEIL((ROC_SG_LIST_HDR_SIZE +\n+\t\t\t (RTE_ALIGN_CEIL(ROC_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SG_ENTRY_SIZE),\n+\t\t\t 8);\n+\n+\treturn len;\n+}\n+\n+static int\n+cnxk_cpt_sec_get_mlen(void)\n+{\n+\tuint32_t len;\n+\n+\tlen = ROC_IE_ON_OUTB_DPTR_HDR + ROC_IE_ON_MAX_IV_LEN;\n+\tlen += RTE_ALIGN_CEIL((ROC_SG_LIST_HDR_SIZE +\n+\t\t\t (RTE_ALIGN_CEIL(ROC_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SG_ENTRY_SIZE),\n+\t\t\t 8);\n \n \treturn len;\n }\n@@ -196,6 +208,11 @@ cnxk_cpt_metabuf_mempool_create(const struct rte_cryptodev *dev,\n \t\tmlen = cnxk_cpt_get_mlen();\n \t}\n \n+\tif (dev->feature_flags & RTE_CRYPTODEV_FF_SECURITY) {\n+\t\t/* Get meta len for security operations */\n+\t\tmlen = cnxk_cpt_sec_get_mlen();\n+\t}\n+\n \tif (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) {\n \n \t\t/* Get meta len required for asymmetric operations */\ndiff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h\nindex 13c90444d6..5153d334ba 100644\n--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h\n+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h\n@@ -158,4 +158,23 @@ pending_queue_free_cnt(uint64_t head, uint64_t tail, const uint64_t mask)\n \treturn mask - pending_queue_infl_cnt(head, tail, mask);\n }\n \n+static __rte_always_inline void *\n+alloc_op_meta(struct roc_se_buf_ptr *buf, int32_t len, struct rte_mempool *cpt_meta_pool,\n+\t struct cpt_inflight_req *infl_req)\n+{\n+\tuint8_t *mdata;\n+\n+\tif (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))\n+\t\treturn NULL;\n+\n+\tif (likely(buf)) {\n+\t\tbuf->vaddr = mdata;\n+\t\tbuf->size = len;\n+\t}\n+\n+\tinfl_req->mdata = mdata;\n+\tinfl_req->op_flags |= CPT_OP_FLAGS_METABUF;\n+\n+\treturn mdata;\n+}\n #endif /* _CNXK_CRYPTODEV_OPS_H_ */\ndiff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h\nindex 2944d3c287..88049ac431 100644\n--- a/drivers/crypto/cnxk/cnxk_se.h\n+++ b/drivers/crypto/cnxk/cnxk_se.h\n@@ -8,11 +8,12 @@\n \n #include \"cnxk_cryptodev.h\"\n #include \"cnxk_cryptodev_ops.h\"\n+#include \"cnxk_sg.h\"\n \n #define SRC_IOV_SIZE \\\n-\t(sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_SE_MAX_SG_CNT))\n+\t(sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_MAX_SG_CNT))\n #define DST_IOV_SIZE \\\n-\t(sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_SE_MAX_SG_CNT))\n+\t(sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_MAX_SG_CNT))\n \n enum cpt_dp_thread_type {\n \tCPT_DP_THREAD_TYPE_FC_CHAIN = 0x1,\n@@ -193,272 +194,14 @@ cpt_fc_salt_update(struct roc_se_ctx *se_ctx, uint8_t *salt)\n \tmemcpy(fctx->enc.encr_iv, salt, 4);\n }\n \n-static __rte_always_inline uint32_t\n-fill_sg_comp(struct roc_se_sglist_comp *list, uint32_t i, phys_addr_t dma_addr,\n-\t uint32_t size)\n-{\n-\tstruct roc_se_sglist_comp *to = &list[i >> 2];\n-\n-\tto->u.s.len[i % 4] = rte_cpu_to_be_16(size);\n-\tto->ptr[i % 4] = rte_cpu_to_be_64(dma_addr);\n-\ti++;\n-\treturn i;\n-}\n-\n-static __rte_always_inline uint32_t\n-fill_sg_comp_from_buf(struct roc_se_sglist_comp *list, uint32_t i,\n-\t\t struct roc_se_buf_ptr *from)\n-{\n-\tstruct roc_se_sglist_comp *to = &list[i >> 2];\n-\n-\tto->u.s.len[i % 4] = rte_cpu_to_be_16(from->size);\n-\tto->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);\n-\ti++;\n-\treturn i;\n-}\n-\n-static __rte_always_inline uint32_t\n-fill_sg_comp_from_buf_min(struct roc_se_sglist_comp *list, uint32_t i,\n-\t\t\t struct roc_se_buf_ptr *from, uint32_t *psize)\n-{\n-\tstruct roc_se_sglist_comp *to = &list[i >> 2];\n-\tuint32_t size = *psize;\n-\tuint32_t e_len;\n-\n-\te_len = (size > from->size) ? from->size : size;\n-\tto->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);\n-\tto->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);\n-\t*psize -= e_len;\n-\ti++;\n-\treturn i;\n-}\n-\n-/*\n- * This fills the MC expected SGIO list\n- * from IOV given by user.\n- */\n-static __rte_always_inline uint32_t\n-fill_sg_comp_from_iov(struct roc_se_sglist_comp *list, uint32_t i,\n-\t\t struct roc_se_iov_ptr *from, uint32_t from_offset,\n-\t\t uint32_t *psize, struct roc_se_buf_ptr *extra_buf,\n-\t\t uint32_t extra_offset)\n-{\n-\tint32_t j;\n-\tuint32_t extra_len = extra_buf ? extra_buf->size : 0;\n-\tuint32_t size = *psize;\n-\n-\tfor (j = 0; j < from->buf_cnt; j++) {\n-\t\tstruct roc_se_sglist_comp *to = &list[i >> 2];\n-\t\tuint32_t buf_sz = from->bufs[j].size;\n-\t\tvoid *vaddr = from->bufs[j].vaddr;\n-\t\tuint64_t e_vaddr;\n-\t\tuint32_t e_len;\n-\n-\t\tif (unlikely(from_offset)) {\n-\t\t\tif (from_offset >= buf_sz) {\n-\t\t\t\tfrom_offset -= buf_sz;\n-\t\t\t\tcontinue;\n-\t\t\t}\n-\t\t\te_vaddr = (uint64_t)vaddr + from_offset;\n-\t\t\te_len = (size > (buf_sz - from_offset)) ?\n-\t\t\t\t\t(buf_sz - from_offset) :\n-\t\t\t\t\tsize;\n-\t\t\tfrom_offset = 0;\n-\t\t} else {\n-\t\t\te_vaddr = (uint64_t)vaddr;\n-\t\t\te_len = (size > buf_sz) ? buf_sz : size;\n-\t\t}\n-\n-\t\tto->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);\n-\t\tto->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr);\n-\n-\t\tif (extra_len && (e_len >= extra_offset)) {\n-\t\t\t/* Break the data at given offset */\n-\t\t\tuint32_t next_len = e_len - extra_offset;\n-\t\t\tuint64_t next_vaddr = e_vaddr + extra_offset;\n-\n-\t\t\tif (!extra_offset) {\n-\t\t\t\ti--;\n-\t\t\t} else {\n-\t\t\t\te_len = extra_offset;\n-\t\t\t\tsize -= e_len;\n-\t\t\t\tto->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);\n-\t\t\t}\n-\n-\t\t\textra_len = RTE_MIN(extra_len, size);\n-\t\t\t/* Insert extra data ptr */\n-\t\t\tif (extra_len) {\n-\t\t\t\ti++;\n-\t\t\t\tto = &list[i >> 2];\n-\t\t\t\tto->u.s.len[i % 4] =\n-\t\t\t\t\trte_cpu_to_be_16(extra_len);\n-\t\t\t\tto->ptr[i % 4] = rte_cpu_to_be_64(\n-\t\t\t\t\t(uint64_t)extra_buf->vaddr);\n-\t\t\t\tsize -= extra_len;\n-\t\t\t}\n-\n-\t\t\tnext_len = RTE_MIN(next_len, size);\n-\t\t\t/* insert the rest of the data */\n-\t\t\tif (next_len) {\n-\t\t\t\ti++;\n-\t\t\t\tto = &list[i >> 2];\n-\t\t\t\tto->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);\n-\t\t\t\tto->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr);\n-\t\t\t\tsize -= next_len;\n-\t\t\t}\n-\t\t\textra_len = 0;\n-\n-\t\t} else {\n-\t\t\tsize -= e_len;\n-\t\t}\n-\t\tif (extra_offset)\n-\t\t\textra_offset -= size;\n-\t\ti++;\n-\n-\t\tif (unlikely(!size))\n-\t\t\tbreak;\n-\t}\n-\n-\t*psize = size;\n-\treturn (uint32_t)i;\n-}\n-\n-static __rte_always_inline uint32_t\n-fill_sg2_comp(struct roc_se_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)\n-{\n-\tstruct roc_se_sg2list_comp *to = &list[i / 3];\n-\n-\tto->u.s.len[i % 3] = (size);\n-\tto->ptr[i % 3] = (dma_addr);\n-\tto->u.s.valid_segs = (i % 3) + 1;\n-\ti++;\n-\treturn i;\n-}\n-\n-static __rte_always_inline uint32_t\n-fill_sg2_comp_from_buf(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from)\n-{\n-\tstruct roc_se_sg2list_comp *to = &list[i / 3];\n-\n-\tto->u.s.len[i % 3] = (from->size);\n-\tto->ptr[i % 3] = ((uint64_t)from->vaddr);\n-\tto->u.s.valid_segs = (i % 3) + 1;\n-\ti++;\n-\treturn i;\n-}\n-\n-static __rte_always_inline uint32_t\n-fill_sg2_comp_from_buf_min(struct roc_se_sg2list_comp *list, uint32_t i,\n-\t\t\t struct roc_se_buf_ptr *from, uint32_t *psize)\n-{\n-\tstruct roc_se_sg2list_comp *to = &list[i / 3];\n-\tuint32_t size = *psize;\n-\tuint32_t e_len;\n-\n-\te_len = (size > from->size) ? from->size : size;\n-\tto->u.s.len[i % 3] = (e_len);\n-\tto->ptr[i % 3] = ((uint64_t)from->vaddr);\n-\tto->u.s.valid_segs = (i % 3) + 1;\n-\t*psize -= e_len;\n-\ti++;\n-\treturn i;\n-}\n-\n-static __rte_always_inline uint32_t\n-fill_sg2_comp_from_iov(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_se_iov_ptr *from,\n-\t\t uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf,\n-\t\t uint32_t extra_offset)\n-{\n-\tint32_t j;\n-\tuint32_t extra_len = extra_buf ? extra_buf->size : 0;\n-\tuint32_t size = *psize;\n-\n-\trte_prefetch2(psize);\n-\n-\tfor (j = 0; j < from->buf_cnt; j++) {\n-\t\tstruct roc_se_sg2list_comp *to = &list[i / 3];\n-\t\tuint32_t buf_sz = from->bufs[j].size;\n-\t\tvoid *vaddr = from->bufs[j].vaddr;\n-\t\tuint64_t e_vaddr;\n-\t\tuint32_t e_len;\n-\n-\t\tif (unlikely(from_offset)) {\n-\t\t\tif (from_offset >= buf_sz) {\n-\t\t\t\tfrom_offset -= buf_sz;\n-\t\t\t\tcontinue;\n-\t\t\t}\n-\t\t\te_vaddr = (uint64_t)vaddr + from_offset;\n-\t\t\te_len = (size > (buf_sz - from_offset)) ? (buf_sz - from_offset) : size;\n-\t\t\tfrom_offset = 0;\n-\t\t} else {\n-\t\t\te_vaddr = (uint64_t)vaddr;\n-\t\t\te_len = (size > buf_sz) ? buf_sz : size;\n-\t\t}\n-\n-\t\tto->u.s.len[i % 3] = (e_len);\n-\t\tto->ptr[i % 3] = (e_vaddr);\n-\t\tto->u.s.valid_segs = (i % 3) + 1;\n-\n-\t\tif (extra_len && (e_len >= extra_offset)) {\n-\t\t\t/* Break the data at given offset */\n-\t\t\tuint32_t next_len = e_len - extra_offset;\n-\t\t\tuint64_t next_vaddr = e_vaddr + extra_offset;\n-\n-\t\t\tif (!extra_offset) {\n-\t\t\t\ti--;\n-\t\t\t} else {\n-\t\t\t\te_len = extra_offset;\n-\t\t\t\tsize -= e_len;\n-\t\t\t\tto->u.s.len[i % 3] = (e_len);\n-\t\t\t}\n-\n-\t\t\textra_len = RTE_MIN(extra_len, size);\n-\t\t\t/* Insert extra data ptr */\n-\t\t\tif (extra_len) {\n-\t\t\t\ti++;\n-\t\t\t\tto = &list[i / 3];\n-\t\t\t\tto->u.s.len[i % 3] = (extra_len);\n-\t\t\t\tto->ptr[i % 3] = ((uint64_t)extra_buf->vaddr);\n-\t\t\t\tto->u.s.valid_segs = (i % 3) + 1;\n-\t\t\t\tsize -= extra_len;\n-\t\t\t}\n-\n-\t\t\tnext_len = RTE_MIN(next_len, size);\n-\t\t\t/* insert the rest of the data */\n-\t\t\tif (next_len) {\n-\t\t\t\ti++;\n-\t\t\t\tto = &list[i / 3];\n-\t\t\t\tto->u.s.len[i % 3] = (next_len);\n-\t\t\t\tto->ptr[i % 3] = (next_vaddr);\n-\t\t\t\tto->u.s.valid_segs = (i % 3) + 1;\n-\t\t\t\tsize -= next_len;\n-\t\t\t}\n-\t\t\textra_len = 0;\n-\n-\t\t} else {\n-\t\t\tsize -= e_len;\n-\t\t}\n-\t\tif (extra_offset)\n-\t\t\textra_offset -= size;\n-\t\ti++;\n-\n-\t\tif (unlikely(!size))\n-\t\t\tbreak;\n-\t}\n-\n-\t*psize = size;\n-\treturn (uint32_t)i;\n-}\n-\n static __rte_always_inline int\n sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t offset_ctrl,\n \t uint8_t *iv_s, int iv_len, uint8_t pack_iv, uint8_t pdcp_alg_type, int32_t inputlen,\n \t int32_t outputlen, uint32_t passthrough_len, uint32_t req_flags, int pdcp_flag,\n \t int decrypt)\n {\n+\tstruct roc_sglist_comp *gather_comp, *scatter_comp;\n \tvoid *m_vaddr = params->meta_buf.vaddr;\n-\tstruct roc_se_sglist_comp *gather_comp;\n-\tstruct roc_se_sglist_comp *scatter_comp;\n \tstruct roc_se_buf_ptr *aad_buf = NULL;\n \tuint32_t mac_len = 0, aad_len = 0;\n \tstruct roc_se_ctx *se_ctx;\n@@ -485,7 +228,7 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t\n \n \tm_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(iv_len, 8);\n \n-\tinst->w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;\n+\tinst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;\n \n \t/* iv offset is 0 */\n \t*offset_vaddr = offset_ctrl;\n@@ -503,7 +246,7 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t\n \t/* DPTR has SG list */\n \n \t/* TODO Add error check if space will be sufficient */\n-\tgather_comp = (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);\n+\tgather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8);\n \n \t/*\n \t * Input Gather List\n@@ -562,13 +305,13 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t\n \t((uint16_t *)in_buffer)[1] = 0;\n \t((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);\n \n-\tg_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);\n+\tg_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n \t/*\n \t * Output Scatter List\n \t */\n \n \ti = 0;\n-\tscatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);\n+\tscatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);\n \n \tif (zsk_flags == 0x1) {\n \t\t/* IV in SLIST only for EEA3 & UEA2 or for F8 */\n@@ -626,9 +369,9 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t\n \t\t}\n \t}\n \t((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);\n-\ts_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);\n+\ts_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n \n-\tsize = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;\n+\tsize = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;\n \n \t/* This is DPTR len in case of SG mode */\n \tinst->w4.s.dlen = size;\n@@ -643,18 +386,17 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t\n \t int32_t outputlen, uint32_t passthrough_len, uint32_t req_flags, int pdcp_flag,\n \t int decrypt)\n {\n+\tstruct roc_sg2list_comp *gather_comp, *scatter_comp;\n \tvoid *m_vaddr = params->meta_buf.vaddr;\n-\tuint32_t i, g_size_bytes;\n-\tstruct roc_se_sg2list_comp *gather_comp;\n-\tstruct roc_se_sg2list_comp *scatter_comp;\n \tstruct roc_se_buf_ptr *aad_buf = NULL;\n+\tuint32_t mac_len = 0, aad_len = 0;\n+\tunion cpt_inst_w5 cpt_inst_w5;\n+\tunion cpt_inst_w6 cpt_inst_w6;\n \tstruct roc_se_ctx *se_ctx;\n+\tuint32_t i, g_size_bytes;\n \tuint64_t *offset_vaddr;\n-\tuint32_t mac_len = 0, aad_len = 0;\n \tint zsk_flags;\n \tuint32_t size;\n-\tunion cpt_inst_w5 cpt_inst_w5;\n-\tunion cpt_inst_w6 cpt_inst_w6;\n \tuint8_t *iv_d;\n \n \tse_ctx = params->ctx;\n@@ -672,7 +414,7 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t\n \n \tm_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(iv_len, 8);\n \n-\tinst->w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;\n+\tinst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;\n \n \t/* iv offset is 0 */\n \t*offset_vaddr = offset_ctrl;\n@@ -689,7 +431,7 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t\n \t/* DPTR has SG list */\n \n \t/* TODO Add error check if space will be sufficient */\n-\tgather_comp = (struct roc_se_sg2list_comp *)((uint8_t *)m_vaddr);\n+\tgather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_vaddr);\n \n \t/*\n \t * Input Gather List\n@@ -746,13 +488,13 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t\n \n \tcpt_inst_w5.s.gather_sz = ((i + 2) / 3);\n \n-\tg_size_bytes = ((i + 2) / 3) * sizeof(struct roc_se_sg2list_comp);\n+\tg_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);\n \t/*\n \t * Output Scatter List\n \t */\n \n \ti = 0;\n-\tscatter_comp = (struct roc_se_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);\n+\tscatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);\n \n \tif (zsk_flags == 0x1) {\n \t\t/* IV in SLIST only for EEA3 & UEA2 or for F8 */\n@@ -829,16 +571,15 @@ static __rte_always_inline int\n cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_params *params,\n \t\t\t struct cpt_inst_s *inst)\n {\n+\tstruct roc_sglist_comp *gather_comp, *scatter_comp;\n \tvoid *m_vaddr = params->meta_buf.vaddr;\n-\tuint32_t size, i;\n+\tuint32_t g_size_bytes, s_size_bytes;\n \tuint16_t data_len, mac_len, key_len;\n+\tunion cpt_inst_w4 cpt_inst_w4;\n \troc_se_auth_type hash_type;\n \tstruct roc_se_ctx *ctx;\n-\tstruct roc_se_sglist_comp *gather_comp;\n-\tstruct roc_se_sglist_comp *scatter_comp;\n \tuint8_t *in_buffer;\n-\tuint32_t g_size_bytes, s_size_bytes;\n-\tunion cpt_inst_w4 cpt_inst_w4;\n+\tuint32_t size, i;\n \n \tctx = params->ctx;\n \n@@ -851,13 +592,11 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa\n \tcpt_inst_w4.s.opcode_minor = 0;\n \tcpt_inst_w4.s.param2 = ((uint16_t)hash_type << 8) | mac_len;\n \tif (ctx->hmac) {\n-\t\tcpt_inst_w4.s.opcode_major =\n-\t\t\tROC_SE_MAJOR_OP_HMAC | ROC_SE_DMA_MODE;\n+\t\tcpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HMAC | ROC_DMA_MODE_SG;\n \t\tcpt_inst_w4.s.param1 = key_len;\n \t\tcpt_inst_w4.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8);\n \t} else {\n-\t\tcpt_inst_w4.s.opcode_major =\n-\t\t\tROC_SE_MAJOR_OP_HASH | ROC_SE_DMA_MODE;\n+\t\tcpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HASH | ROC_DMA_MODE_SG;\n \t\tcpt_inst_w4.s.param1 = 0;\n \t\tcpt_inst_w4.s.dlen = data_len;\n \t}\n@@ -878,7 +617,7 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa\n \t((uint16_t *)in_buffer)[1] = 0;\n \n \t/* TODO Add error check if space will be sufficient */\n-\tgather_comp = (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);\n+\tgather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8);\n \n \t/*\n \t * Input gather list\n@@ -901,15 +640,14 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa\n \t\treturn -1;\n \t}\n \t((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);\n-\tg_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);\n+\tg_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n \n \t/*\n \t * Output Gather list\n \t */\n \n \ti = 0;\n-\tscatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp +\n-\t\t\t\t\t\t g_size_bytes);\n+\tscatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);\n \n \tif (flags & ROC_SE_VALID_MAC_BUF) {\n \t\tif (unlikely(params->mac_buf.size < mac_len)) {\n@@ -932,9 +670,9 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa\n \t}\n \n \t((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);\n-\ts_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);\n+\ts_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n \n-\tsize = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;\n+\tsize = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;\n \n \t/* This is DPTR len in case of SG mode */\n \tcpt_inst_w4.s.dlen = size;\n@@ -949,17 +687,16 @@ static __rte_always_inline int\n cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_params *params,\n \t\t\t struct cpt_inst_s *inst)\n {\n+\tstruct roc_sg2list_comp *gather_comp, *scatter_comp;\n \tvoid *m_vaddr = params->meta_buf.vaddr;\n-\tuint32_t size, i;\n \tuint16_t data_len, mac_len, key_len;\n-\troc_se_auth_type hash_type;\n-\tstruct roc_se_ctx *ctx;\n-\tstruct roc_se_sg2list_comp *gather_comp;\n-\tstruct roc_se_sg2list_comp *scatter_comp;\n+\tunion cpt_inst_w4 cpt_inst_w4;\n \tunion cpt_inst_w5 cpt_inst_w5;\n \tunion cpt_inst_w6 cpt_inst_w6;\n+\troc_se_auth_type hash_type;\n+\tstruct roc_se_ctx *ctx;\n \tuint32_t g_size_bytes;\n-\tunion cpt_inst_w4 cpt_inst_w4;\n+\tuint32_t size, i;\n \n \tctx = params->ctx;\n \n@@ -993,7 +730,7 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa\n \t/* DPTR has SG list */\n \n \t/* TODO Add error check if space will be sufficient */\n-\tgather_comp = (struct roc_se_sg2list_comp *)((uint8_t *)m_vaddr + 0);\n+\tgather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_vaddr + 0);\n \n \t/*\n \t * Input gather list\n@@ -1016,14 +753,14 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa\n \t}\n \tcpt_inst_w5.s.gather_sz = ((i + 2) / 3);\n \n-\tg_size_bytes = ((i + 2) / 3) * sizeof(struct roc_se_sg2list_comp);\n+\tg_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);\n \n \t/*\n \t * Output Gather list\n \t */\n \n \ti = 0;\n-\tscatter_comp = (struct roc_se_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);\n+\tscatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);\n \n \tif (flags & ROC_SE_VALID_MAC_BUF) {\n \t\tif (unlikely(params->mac_buf.size < mac_len)) {\n@@ -1482,8 +1219,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,\n \t\tpdcp_iv_copy(iv_d, auth_iv, pdcp_auth_alg, pack_iv);\n \n \t} else {\n-\n-\t\tstruct roc_se_sglist_comp *scatter_comp, *gather_comp;\n+\t\tstruct roc_sglist_comp *scatter_comp, *gather_comp;\n \t\tvoid *m_vaddr = params->meta_buf.vaddr;\n \t\tuint32_t i, g_size_bytes, s_size_bytes;\n \t\tuint8_t *in_buffer;\n@@ -1494,7 +1230,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,\n \n \t\tm_vaddr = PLT_PTR_ADD(m_vaddr, ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(hdr_len, 8));\n \n-\t\tcpt_inst_w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;\n+\t\tcpt_inst_w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;\n \n \t\t/* DPTR has SG list */\n \t\tin_buffer = m_vaddr;\n@@ -1502,8 +1238,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,\n \t\t((uint16_t *)in_buffer)[0] = 0;\n \t\t((uint16_t *)in_buffer)[1] = 0;\n \n-\t\tgather_comp =\n-\t\t\t(struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);\n+\t\tgather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8);\n \n \t\t/* Input Gather List */\n \t\ti = 0;\n@@ -1536,15 +1271,14 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,\n \t\t\t}\n \t\t}\n \t\t((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);\n-\t\tg_size_bytes =\n-\t\t\t((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);\n+\t\tg_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n \n \t\t/*\n \t\t * Output Scatter List\n \t\t */\n \n \t\ti = 0;\n-\t\tscatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);\n+\t\tscatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);\n \n \t\tif ((hdr_len)) {\n \t\t\ti = fill_sg_comp(scatter_comp, i,\n@@ -1573,10 +1307,9 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,\n \t\t}\n \n \t\t((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);\n-\t\ts_size_bytes =\n-\t\t\t((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);\n+\t\ts_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);\n \n-\t\tsize = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;\n+\t\tsize = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;\n \n \t\t/* This is DPTR len in case of SG mode */\n \t\tcpt_inst_w4.s.dlen = size;\n@@ -1800,7 +1533,7 @@ cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,\n \t\t}\n \t}\n \n-\tcpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_SE_DMA_MODE;\n+\tcpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_DMA_MODE_SG;\n \n \t/* Indicate ECB/CBC, direction, CTX from CPTR, IV from DPTR */\n \tcpt_inst_w4.s.opcode_minor =\n@@ -1841,11 +1574,11 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t d_lens, struct roc_se_fc_params *p\n \tflags = se_ctx->zsk_flags;\n \n \tcpt_inst_w4.u64 = 0;\n-\tcpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_SE_DMA_MODE;\n+\tcpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_DMA_MODE_SG;\n \n \t/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */\n-\tcpt_inst_w4.s.opcode_minor = ((1 << 6) | (se_ctx->k_ecb << 5) |\n-\t\t\t\t (dir << 4) | (0 << 3) | (flags & 0x7));\n+\tcpt_inst_w4.s.opcode_minor =\n+\t\t((1 << 6) | (se_ctx->k_ecb << 5) | (dir << 4) | (0 << 3) | (flags & 0x7));\n \n \t/*\n \t * GP op header, lengths are expected in bits.\n@@ -2290,28 +2023,8 @@ fill_sess_gmac(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)\n \treturn 0;\n }\n \n-static __rte_always_inline void *\n-alloc_op_meta(struct roc_se_buf_ptr *buf, int32_t len,\n-\t struct rte_mempool *cpt_meta_pool,\n-\t struct cpt_inflight_req *infl_req)\n-{\n-\tuint8_t *mdata;\n-\n-\tif (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))\n-\t\treturn NULL;\n-\n-\tbuf->vaddr = mdata;\n-\tbuf->size = len;\n-\n-\tinfl_req->mdata = mdata;\n-\tinfl_req->op_flags |= CPT_OP_FLAGS_METABUF;\n-\n-\treturn mdata;\n-}\n-\n static __rte_always_inline uint32_t\n-prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec,\n-\t\t uint32_t start_offset)\n+prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec, uint32_t start_offset)\n {\n \tuint16_t index = 0;\n \tvoid *seg_data = NULL;\ndiff --git a/drivers/crypto/cnxk/cnxk_sg.h b/drivers/crypto/cnxk/cnxk_sg.h\nnew file mode 100644\nindex 0000000000..1dfca261cf\n--- /dev/null\n+++ b/drivers/crypto/cnxk/cnxk_sg.h\n@@ -0,0 +1,273 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(C) 2022 Marvell.\n+ */\n+\n+#ifndef _CNXK_SG_H_\n+#define _CNXK_SG_H_\n+\n+static __rte_always_inline uint32_t\n+fill_sg_comp(struct roc_sglist_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)\n+{\n+\tstruct roc_sglist_comp *to = &list[i >> 2];\n+\n+\tto->u.s.len[i % 4] = rte_cpu_to_be_16(size);\n+\tto->ptr[i % 4] = rte_cpu_to_be_64(dma_addr);\n+\treturn ++i;\n+}\n+\n+static __rte_always_inline uint32_t\n+fill_sg_comp_from_buf(struct roc_sglist_comp *list, uint32_t i, struct roc_se_buf_ptr *from)\n+{\n+\tstruct roc_sglist_comp *to = &list[i >> 2];\n+\n+\tto->u.s.len[i % 4] = rte_cpu_to_be_16(from->size);\n+\tto->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);\n+\treturn ++i;\n+}\n+\n+static __rte_always_inline uint32_t\n+fill_sg_comp_from_buf_min(struct roc_sglist_comp *list, uint32_t i, struct roc_se_buf_ptr *from,\n+\t\t\t uint32_t *psize)\n+{\n+\tstruct roc_sglist_comp *to = &list[i >> 2];\n+\tuint32_t size = *psize;\n+\tuint32_t e_len;\n+\n+\te_len = RTE_MIN(from->size, size);\n+\tto->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);\n+\tto->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);\n+\t*psize -= e_len;\n+\treturn ++i;\n+}\n+\n+/*\n+ * This fills the MC expected SGIO list\n+ * from IOV given by user.\n+ */\n+static __rte_always_inline uint32_t\n+fill_sg_comp_from_iov(struct roc_sglist_comp *list, uint32_t i, struct roc_se_iov_ptr *from,\n+\t\t uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf,\n+\t\t uint32_t extra_offset)\n+{\n+\tuint32_t extra_len = extra_buf ? extra_buf->size : 0;\n+\tuint32_t size = *psize;\n+\tint32_t j;\n+\n+\tfor (j = 0; j < from->buf_cnt; j++) {\n+\t\tstruct roc_sglist_comp *to = &list[i >> 2];\n+\t\tuint32_t buf_sz = from->bufs[j].size;\n+\t\tvoid *vaddr = from->bufs[j].vaddr;\n+\t\tuint64_t e_vaddr;\n+\t\tuint32_t e_len;\n+\n+\t\tif (unlikely(from_offset)) {\n+\t\t\tif (from_offset >= buf_sz) {\n+\t\t\t\tfrom_offset -= buf_sz;\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\te_vaddr = (uint64_t)vaddr + from_offset;\n+\t\t\te_len = RTE_MIN((buf_sz - from_offset), size);\n+\t\t\tfrom_offset = 0;\n+\t\t} else {\n+\t\t\te_vaddr = (uint64_t)vaddr;\n+\t\t\te_len = RTE_MIN(buf_sz, size);\n+\t\t}\n+\n+\t\tto->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);\n+\t\tto->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr);\n+\n+\t\tif (extra_len && (e_len >= extra_offset)) {\n+\t\t\t/* Break the data at given offset */\n+\t\t\tuint32_t next_len = e_len - extra_offset;\n+\t\t\tuint64_t next_vaddr = e_vaddr + extra_offset;\n+\n+\t\t\tif (!extra_offset) {\n+\t\t\t\ti--;\n+\t\t\t} else {\n+\t\t\t\te_len = extra_offset;\n+\t\t\t\tsize -= e_len;\n+\t\t\t\tto->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);\n+\t\t\t}\n+\n+\t\t\textra_len = RTE_MIN(extra_len, size);\n+\t\t\t/* Insert extra data ptr */\n+\t\t\tif (extra_len) {\n+\t\t\t\ti++;\n+\t\t\t\tto = &list[i >> 2];\n+\t\t\t\tto->u.s.len[i % 4] = rte_cpu_to_be_16(extra_len);\n+\t\t\t\tto->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)extra_buf->vaddr);\n+\t\t\t\tsize -= extra_len;\n+\t\t\t}\n+\n+\t\t\tnext_len = RTE_MIN(next_len, size);\n+\t\t\t/* insert the rest of the data */\n+\t\t\tif (next_len) {\n+\t\t\t\ti++;\n+\t\t\t\tto = &list[i >> 2];\n+\t\t\t\tto->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);\n+\t\t\t\tto->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr);\n+\t\t\t\tsize -= next_len;\n+\t\t\t}\n+\t\t\textra_len = 0;\n+\n+\t\t} else {\n+\t\t\tsize -= e_len;\n+\t\t}\n+\t\tif (extra_offset)\n+\t\t\textra_offset -= size;\n+\t\ti++;\n+\n+\t\tif (unlikely(!size))\n+\t\t\tbreak;\n+\t}\n+\n+\t*psize = size;\n+\treturn (uint32_t)i;\n+}\n+\n+static __rte_always_inline uint32_t\n+fill_ipsec_sg_comp_from_pkt(struct roc_sglist_comp *list, uint32_t i, struct rte_mbuf *pkt)\n+{\n+\tuint32_t buf_sz;\n+\tvoid *vaddr;\n+\n+\twhile (unlikely(pkt != NULL)) {\n+\t\tstruct roc_sglist_comp *to = &list[i >> 2];\n+\t\tbuf_sz = pkt->data_len;\n+\t\tvaddr = rte_pktmbuf_mtod(pkt, void *);\n+\n+\t\tto->u.s.len[i % 4] = rte_cpu_to_be_16(buf_sz);\n+\t\tto->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)vaddr);\n+\n+\t\tpkt = pkt->next;\n+\t\ti++;\n+\t}\n+\n+\treturn i;\n+}\n+\n+static __rte_always_inline uint32_t\n+fill_sg2_comp(struct roc_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)\n+{\n+\tstruct roc_sg2list_comp *to = &list[i / 3];\n+\n+\tto->u.s.len[i % 3] = (size);\n+\tto->ptr[i % 3] = (dma_addr);\n+\tto->u.s.valid_segs = (i % 3) + 1;\n+\treturn ++i;\n+}\n+\n+static __rte_always_inline uint32_t\n+fill_sg2_comp_from_buf(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from)\n+{\n+\tstruct roc_sg2list_comp *to = &list[i / 3];\n+\n+\tto->u.s.len[i % 3] = (from->size);\n+\tto->ptr[i % 3] = ((uint64_t)from->vaddr);\n+\tto->u.s.valid_segs = (i % 3) + 1;\n+\treturn ++i;\n+}\n+\n+static __rte_always_inline uint32_t\n+fill_sg2_comp_from_buf_min(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from,\n+\t\t\t uint32_t *psize)\n+{\n+\tstruct roc_sg2list_comp *to = &list[i / 3];\n+\tuint32_t size = *psize;\n+\tuint32_t e_len;\n+\n+\te_len = RTE_MIN(from->size, size);\n+\tto->u.s.len[i % 3] = (e_len);\n+\tto->ptr[i % 3] = ((uint64_t)from->vaddr);\n+\tto->u.s.valid_segs = (i % 3) + 1;\n+\t*psize -= e_len;\n+\treturn ++i;\n+}\n+\n+static __rte_always_inline uint32_t\n+fill_sg2_comp_from_iov(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_iov_ptr *from,\n+\t\t uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf,\n+\t\t uint32_t extra_offset)\n+{\n+\tuint32_t extra_len = extra_buf ? extra_buf->size : 0;\n+\tuint32_t size = *psize;\n+\tint32_t j;\n+\n+\trte_prefetch2(psize);\n+\n+\tfor (j = 0; j < from->buf_cnt; j++) {\n+\t\tstruct roc_sg2list_comp *to = &list[i / 3];\n+\t\tuint32_t buf_sz = from->bufs[j].size;\n+\t\tvoid *vaddr = from->bufs[j].vaddr;\n+\t\tuint64_t e_vaddr;\n+\t\tuint32_t e_len;\n+\n+\t\tif (unlikely(from_offset)) {\n+\t\t\tif (from_offset >= buf_sz) {\n+\t\t\t\tfrom_offset -= buf_sz;\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\te_vaddr = (uint64_t)vaddr + from_offset;\n+\t\t\te_len = RTE_MIN((buf_sz - from_offset), size);\n+\t\t\tfrom_offset = 0;\n+\t\t} else {\n+\t\t\te_vaddr = (uint64_t)vaddr;\n+\t\t\te_len = RTE_MIN(buf_sz, size);\n+\t\t}\n+\n+\t\tto->u.s.len[i % 3] = (e_len);\n+\t\tto->ptr[i % 3] = (e_vaddr);\n+\t\tto->u.s.valid_segs = (i % 3) + 1;\n+\n+\t\tif (extra_len && (e_len >= extra_offset)) {\n+\t\t\t/* Break the data at given offset */\n+\t\t\tuint32_t next_len = e_len - extra_offset;\n+\t\t\tuint64_t next_vaddr = e_vaddr + extra_offset;\n+\n+\t\t\tif (!extra_offset) {\n+\t\t\t\ti--;\n+\t\t\t} else {\n+\t\t\t\te_len = extra_offset;\n+\t\t\t\tsize -= e_len;\n+\t\t\t\tto->u.s.len[i % 3] = (e_len);\n+\t\t\t}\n+\n+\t\t\textra_len = RTE_MIN(extra_len, size);\n+\t\t\t/* Insert extra data ptr */\n+\t\t\tif (extra_len) {\n+\t\t\t\ti++;\n+\t\t\t\tto = &list[i / 3];\n+\t\t\t\tto->u.s.len[i % 3] = (extra_len);\n+\t\t\t\tto->ptr[i % 3] = ((uint64_t)extra_buf->vaddr);\n+\t\t\t\tto->u.s.valid_segs = (i % 3) + 1;\n+\t\t\t\tsize -= extra_len;\n+\t\t\t}\n+\n+\t\t\tnext_len = RTE_MIN(next_len, size);\n+\t\t\t/* insert the rest of the data */\n+\t\t\tif (next_len) {\n+\t\t\t\ti++;\n+\t\t\t\tto = &list[i / 3];\n+\t\t\t\tto->u.s.len[i % 3] = (next_len);\n+\t\t\t\tto->ptr[i % 3] = (next_vaddr);\n+\t\t\t\tto->u.s.valid_segs = (i % 3) + 1;\n+\t\t\t\tsize -= next_len;\n+\t\t\t}\n+\t\t\textra_len = 0;\n+\n+\t\t} else {\n+\t\t\tsize -= e_len;\n+\t\t}\n+\t\tif (extra_offset)\n+\t\t\textra_offset -= size;\n+\t\ti++;\n+\n+\t\tif (unlikely(!size))\n+\t\t\tbreak;\n+\t}\n+\n+\t*psize = size;\n+\treturn (uint32_t)i;\n+}\n+\n+#endif /*_CNXK_SG_H_ */\n", "prefixes": [ "09/17" ] }