Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 75641,
    "url": "http://patches.dpdk.org/api/patches/75641/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/20200818162833.20219-3-roy.fan.zhang@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20200818162833.20219-3-roy.fan.zhang@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20200818162833.20219-3-roy.fan.zhang@intel.com",
    "date": "2020-08-18T16:28:31",
    "name": "[v6,2/4] crypto/qat: add crypto data-path service API support",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "e1d0bde7f8fdf58b85b51bfa102c37fb0d036968",
    "submitter": {
        "id": 304,
        "url": "http://patches.dpdk.org/api/people/304/?format=api",
        "name": "Fan Zhang",
        "email": "roy.fan.zhang@intel.com"
    },
    "delegate": {
        "id": 6690,
        "url": "http://patches.dpdk.org/api/users/6690/?format=api",
        "username": "akhil",
        "first_name": "akhil",
        "last_name": "goyal",
        "email": "gakhil@marvell.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/20200818162833.20219-3-roy.fan.zhang@intel.com/mbox/",
    "series": [
        {
            "id": 11694,
            "url": "http://patches.dpdk.org/api/series/11694/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=11694",
            "date": "2020-08-18T16:28:29",
            "name": "cryptodev: add data-path service APIs",
            "version": 6,
            "mbox": "http://patches.dpdk.org/series/11694/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/75641/comments/",
    "check": "warning",
    "checks": "http://patches.dpdk.org/api/patches/75641/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 29CDDA0353;\n\tTue, 18 Aug 2020 18:29:16 +0200 (CEST)",
            "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 6E6B81C037;\n\tTue, 18 Aug 2020 18:28:54 +0200 (CEST)",
            "from mga17.intel.com (mga17.intel.com [192.55.52.151])\n by dpdk.org (Postfix) with ESMTP id E38512A62\n for <dev@dpdk.org>; Tue, 18 Aug 2020 18:28:49 +0200 (CEST)",
            "from fmsmga004.fm.intel.com ([10.253.24.48])\n by fmsmga107.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 18 Aug 2020 09:28:41 -0700",
            "from silpixa00398673.ir.intel.com (HELO\n silpixa00398673.ger.corp.intel.com) ([10.237.223.136])\n by fmsmga004.fm.intel.com with ESMTP; 18 Aug 2020 09:28:39 -0700"
        ],
        "IronPort-SDR": [
            "\n EjLDDmNncvEbmGh+M7FW1r6qYorXjXvctUHzgaYO30uxyeagyVbh0s2SNCDNpbP+8LhzAbyQGT\n ahEhA1jIbR5A==",
            "\n 8d7zsgs4QTFFQQ8nVhzf8e6RMGXUPFZfCZiQf/0lHhT51iP7kY741vpRv9EignVVM+VEQrYq1A\n d7K2mYrUWHAQ=="
        ],
        "X-IronPort-AV": [
            "E=McAfee;i=\"6000,8403,9716\"; a=\"135011828\"",
            "E=Sophos;i=\"5.76,328,1592895600\"; d=\"scan'208\";a=\"135011828\"",
            "E=Sophos;i=\"5.76,328,1592895600\"; d=\"scan'208\";a=\"320148790\""
        ],
        "X-Amp-Result": "SKIPPED(no attachment in message)",
        "X-Amp-File-Uploaded": "False",
        "X-ExtLoop1": "1",
        "From": "Fan Zhang <roy.fan.zhang@intel.com>",
        "To": "dev@dpdk.org",
        "Cc": "akhil.goyal@nxp.com, fiona.trahe@intel.com, ArkadiuszX.Kusztal@intel.com,\n AdamX.Dybkowski@intel.com, Fan Zhang <roy.fan.zhang@intel.com>",
        "Date": "Tue, 18 Aug 2020 17:28:31 +0100",
        "Message-Id": "<20200818162833.20219-3-roy.fan.zhang@intel.com>",
        "X-Mailer": "git-send-email 2.20.1",
        "In-Reply-To": "<20200818162833.20219-1-roy.fan.zhang@intel.com>",
        "References": "<20200818162833.20219-1-roy.fan.zhang@intel.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "Subject": "[dpdk-dev] [dpdk-dev v6 2/4] crypto/qat: add crypto data-path\n\tservice API support",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.15",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "This patch updates QAT PMD to add crypto service API support.\n\nSigned-off-by: Fan Zhang <roy.fan.zhang@intel.com>\n---\n drivers/common/qat/Makefile        |   1 +\n drivers/crypto/qat/meson.build     |   1 +\n drivers/crypto/qat/qat_sym.h       |  13 +\n drivers/crypto/qat/qat_sym_hw_dp.c | 923 +++++++++++++++++++++++++++++\n drivers/crypto/qat/qat_sym_pmd.c   |   9 +-\n 5 files changed, 945 insertions(+), 2 deletions(-)\n create mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c",
    "diff": "diff --git a/drivers/common/qat/Makefile b/drivers/common/qat/Makefile\nindex 85d420709..1b71bbbab 100644\n--- a/drivers/common/qat/Makefile\n+++ b/drivers/common/qat/Makefile\n@@ -42,6 +42,7 @@ endif\n \tSRCS-y += qat_sym.c\n \tSRCS-y += qat_sym_session.c\n \tSRCS-y += qat_sym_pmd.c\n+\tSRCS-y += qat_sym_hw_dp.c\n \tbuild_qat = yes\n endif\n endif\ndiff --git a/drivers/crypto/qat/meson.build b/drivers/crypto/qat/meson.build\nindex a225f374a..bc90ec44c 100644\n--- a/drivers/crypto/qat/meson.build\n+++ b/drivers/crypto/qat/meson.build\n@@ -15,6 +15,7 @@ if dep.found()\n \tqat_sources += files('qat_sym_pmd.c',\n \t\t\t     'qat_sym.c',\n \t\t\t     'qat_sym_session.c',\n+\t\t\t     'qat_sym_hw_dp.c',\n \t\t\t     'qat_asym_pmd.c',\n \t\t\t     'qat_asym.c')\n \tqat_ext_deps += dep\ndiff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h\nindex 1a9748849..2d6316130 100644\n--- a/drivers/crypto/qat/qat_sym.h\n+++ b/drivers/crypto/qat/qat_sym.h\n@@ -264,6 +264,18 @@ qat_sym_process_response(void **op, uint8_t *resp)\n \t}\n \t*op = (void *)rx_op;\n }\n+\n+int\n+qat_sym_dp_configure_service_ctx(struct rte_cryptodev *dev, uint16_t qp_id,\n+\tstruct rte_crypto_dp_service_ctx *service_ctx,\n+\tenum rte_crypto_dp_service service_type,\n+\tenum rte_crypto_op_sess_type sess_type,\n+\tunion rte_cryptodev_session_ctx session_ctx,\n+\tuint8_t is_update);\n+\n+int\n+qat_sym_get_service_ctx_size(struct rte_cryptodev *dev);\n+\n #else\n \n static inline void\n@@ -276,5 +288,6 @@ static inline void\n qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused)\n {\n }\n+\n #endif\n #endif /* _QAT_SYM_H_ */\ndiff --git a/drivers/crypto/qat/qat_sym_hw_dp.c b/drivers/crypto/qat/qat_sym_hw_dp.c\nnew file mode 100644\nindex 000000000..bb372a0b2\n--- /dev/null\n+++ b/drivers/crypto/qat/qat_sym_hw_dp.c\n@@ -0,0 +1,923 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(c) 2020 Intel Corporation\n+ */\n+\n+#include <rte_cryptodev_pmd.h>\n+\n+#include \"adf_transport_access_macros.h\"\n+#include \"icp_qat_fw.h\"\n+#include \"icp_qat_fw_la.h\"\n+\n+#include \"qat_sym.h\"\n+#include \"qat_sym_pmd.h\"\n+#include \"qat_sym_session.h\"\n+#include \"qat_qp.h\"\n+\n+struct qat_sym_dp_service_ctx {\n+\tstruct qat_sym_session *session;\n+\tuint32_t tail;\n+\tuint32_t head;\n+};\n+\n+static __rte_always_inline int32_t\n+qat_sym_dp_get_data(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,\n+\t\tstruct rte_crypto_vec *data, uint16_t n_data_vecs)\n+{\n+\tstruct qat_queue *tx_queue;\n+\tstruct qat_sym_op_cookie *cookie;\n+\tstruct qat_sgl *list;\n+\tuint32_t i;\n+\tuint32_t total_len;\n+\n+\tif (likely(n_data_vecs == 1)) {\n+\t\treq->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =\n+\t\t\tdata[0].iova;\n+\t\treq->comn_mid.src_length = req->comn_mid.dst_length =\n+\t\t\tdata[0].len;\n+\t\treturn data[0].len;\n+\t}\n+\n+\tif (n_data_vecs == 0 || n_data_vecs > QAT_SYM_SGL_MAX_NUMBER)\n+\t\treturn -1;\n+\n+\ttotal_len = 0;\n+\ttx_queue = &qp->tx_q;\n+\n+\tICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,\n+\t\t\tQAT_COMN_PTR_TYPE_SGL);\n+\tcookie = qp->op_cookies[tx_queue->tail >> tx_queue->trailz];\n+\tlist = (struct qat_sgl *)&cookie->qat_sgl_src;\n+\n+\tfor (i = 0; i < n_data_vecs; i++) {\n+\t\tlist->buffers[i].len = data[i].len;\n+\t\tlist->buffers[i].resrvd = 0;\n+\t\tlist->buffers[i].addr = data[i].iova;\n+\t\tif (total_len + data[i].len > UINT32_MAX) {\n+\t\t\tQAT_DP_LOG(ERR, \"Message too long\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\ttotal_len += data[i].len;\n+\t}\n+\n+\tlist->num_bufs = i;\n+\treq->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =\n+\t\t\tcookie->qat_sgl_src_phys_addr;\n+\treq->comn_mid.src_length = req->comn_mid.dst_length = 0;\n+\treturn total_len;\n+}\n+\n+static __rte_always_inline void\n+set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,\n+\t\tstruct rte_crypto_data *iv, uint32_t iv_len,\n+\t\tstruct icp_qat_fw_la_bulk_req *qat_req)\n+{\n+\t/* copy IV into request if it fits */\n+\tif (iv_len <= sizeof(cipher_param->u.cipher_IV_array))\n+\t\trte_memcpy(cipher_param->u.cipher_IV_array, iv->base, iv_len);\n+\telse {\n+\t\tICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(\n+\t\t\t\tqat_req->comn_hdr.serv_specif_flags,\n+\t\t\t\tICP_QAT_FW_CIPH_IV_64BIT_PTR);\n+\t\tcipher_param->u.s.cipher_IV_ptr = iv->iova;\n+\t}\n+}\n+\n+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \\\n+\t(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \\\n+\tICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))\n+\n+static __rte_always_inline void\n+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)\n+{\n+\tuint32_t i;\n+\n+\tfor (i = 0; i < n; i++)\n+\t\tsta[i] = status;\n+}\n+\n+static __rte_always_inline void\n+submit_one_aead_job(struct qat_sym_session *ctx,\n+\tstruct icp_qat_fw_la_bulk_req *req, struct rte_crypto_data *iv_vec,\n+\tstruct rte_crypto_data *digest_vec, struct rte_crypto_data *aad_vec,\n+\tunion rte_crypto_sym_ofs ofs, uint32_t data_len)\n+{\n+\tstruct icp_qat_fw_la_cipher_req_params *cipher_param =\n+\t\t(void *)&req->serv_specif_rqpars;\n+\tstruct icp_qat_fw_la_auth_req_params *auth_param =\n+\t\t(void *)((uint8_t *)&req->serv_specif_rqpars +\n+\t\tICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);\n+\tuint8_t *aad_data;\n+\tuint8_t aad_ccm_real_len;\n+\tuint8_t aad_len_field_sz;\n+\tuint32_t msg_len_be;\n+\trte_iova_t aad_iova = 0;\n+\tuint8_t q;\n+\n+\tswitch (ctx->qat_hash_alg) {\n+\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_128:\n+\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_64:\n+\t\tICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(\n+\t\t\treq->comn_hdr.serv_specif_flags,\n+\t\t\t\tICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);\n+\t\trte_memcpy_generic(cipher_param->u.cipher_IV_array,\n+\t\t\t\tiv_vec->base, ctx->cipher_iv.length);\n+\t\taad_iova = aad_vec->iova;\n+\t\tbreak;\n+\tcase ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:\n+\t\taad_data = aad_vec->base;\n+\t\taad_iova = aad_vec->iova;\n+\t\taad_ccm_real_len = 0;\n+\t\taad_len_field_sz = 0;\n+\t\tmsg_len_be = rte_bswap32((uint32_t)data_len -\n+\t\t\t\tofs.ofs.cipher.head);\n+\n+\t\tif (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {\n+\t\t\taad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;\n+\t\t\taad_ccm_real_len = ctx->aad_len -\n+\t\t\t\tICP_QAT_HW_CCM_AAD_B0_LEN -\n+\t\t\t\tICP_QAT_HW_CCM_AAD_LEN_INFO;\n+\t\t} else {\n+\t\t\taad_data = iv_vec->base;\n+\t\t\taad_iova = iv_vec->iova;\n+\t\t}\n+\n+\t\tq = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;\n+\t\taad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(\n+\t\t\taad_len_field_sz, ctx->digest_length, q);\n+\t\tif (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {\n+\t\t\tmemcpy(aad_data\t+ ctx->cipher_iv.length +\n+\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET + (q -\n+\t\t\t\tICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),\n+\t\t\t\t(uint8_t *)&msg_len_be,\n+\t\t\t\tICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);\n+\t\t} else {\n+\t\t\tmemcpy(aad_data\t+ ctx->cipher_iv.length +\n+\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\t\t(uint8_t *)&msg_len_be +\n+\t\t\t\t(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE\n+\t\t\t\t- q), q);\n+\t\t}\n+\n+\t\tif (aad_len_field_sz > 0) {\n+\t\t\t*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =\n+\t\t\t\trte_bswap16(aad_ccm_real_len);\n+\n+\t\t\tif ((aad_ccm_real_len + aad_len_field_sz)\n+\t\t\t\t% ICP_QAT_HW_CCM_AAD_B0_LEN) {\n+\t\t\t\tuint8_t pad_len = 0;\n+\t\t\t\tuint8_t pad_idx = 0;\n+\n+\t\t\t\tpad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -\n+\t\t\t\t\t((aad_ccm_real_len +\n+\t\t\t\t\taad_len_field_sz) %\n+\t\t\t\t\tICP_QAT_HW_CCM_AAD_B0_LEN);\n+\t\t\t\tpad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +\n+\t\t\t\t\taad_ccm_real_len +\n+\t\t\t\t\taad_len_field_sz;\n+\t\t\t\tmemset(&aad_data[pad_idx], 0, pad_len);\n+\t\t\t}\n+\n+\t\t\trte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)\n+\t\t\t\t+ ICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\t\t(uint8_t *)iv_vec->base +\n+\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\t\tctx->cipher_iv.length);\n+\t\t\t*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =\n+\t\t\t\tq - ICP_QAT_HW_CCM_NONCE_OFFSET;\n+\n+\t\t\trte_memcpy((uint8_t *)aad_vec->base +\n+\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\t\t(uint8_t *)iv_vec->base +\n+\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\t\tctx->cipher_iv.length);\n+\t\t}\n+\t\tbreak;\n+\tdefault:\n+\t\tbreak;\n+\t}\n+\n+\tcipher_param->cipher_offset = ofs.ofs.cipher.head;\n+\tcipher_param->cipher_length = data_len - ofs.ofs.cipher.head;\n+\tauth_param->auth_off = ofs.ofs.cipher.head;\n+\tauth_param->auth_len = data_len - ofs.ofs.cipher.head;\n+\tauth_param->auth_res_addr = digest_vec->iova;\n+\tauth_param->u1.aad_adr = aad_iova;\n+\n+\tif (ctx->is_single_pass) {\n+\t\tcipher_param->spc_aad_addr = aad_iova;\n+\t\tcipher_param->spc_auth_res_addr = digest_vec->iova;\n+\t}\n+}\n+\n+static __rte_always_inline int\n+qat_sym_dp_submit_single_aead(void *qp_data, uint8_t *service_data,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs, struct rte_crypto_data *iv_vec,\n+\tstruct rte_crypto_data *digest_vec, struct rte_crypto_data *aad_vec,\n+\tvoid *opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = service_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = service_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_dp_get_data(qp, req, data, n_data_vecs);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)opaque;\n+\n+\tsubmit_one_aead_job(ctx, req, iv_vec, digest_vec, aad_vec, ofs,\n+\t\t(uint32_t)data_len);\n+\n+\tservice_ctx->tail = tail;\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline uint32_t\n+qat_sym_dp_submit_aead_jobs(void *qp_data, uint8_t *service_data,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid **opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = service_ctx->session;\n+\tuint32_t i;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tif (unlikely(qp->enqueued - qp->dequeued + vec->num >=\n+\t\t\tqp->max_inflights)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\treturn 0;\n+\t}\n+\n+\ttail = service_ctx->tail;\n+\n+\tfor (i = 0; i < vec->num; i++) {\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_dp_get_data(qp, req, vec->sgl[i].vec,\n+\t\t\tvec->sgl[i].num) - ofs.ofs.cipher.head -\n+\t\t\tofs.ofs.cipher.tail;\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\t\treq->comn_mid.opaque_data = (uint64_t)opaque[i];\n+\t\tsubmit_one_aead_job(ctx, req, vec->iv_vec + i,\n+\t\t\tvec->digest_vec + i, vec->aad_vec + i, ofs,\n+\t\t\t(uint32_t)data_len);\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\t}\n+\n+\tif (unlikely(i < vec->num))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, vec->num - i);\n+\n+\tservice_ctx->tail = tail;\n+\treturn i;\n+}\n+\n+static __rte_always_inline void\n+submit_one_cipher_job(struct qat_sym_session *ctx,\n+\tstruct icp_qat_fw_la_bulk_req *req, struct rte_crypto_data *iv_vec,\n+\tunion rte_crypto_sym_ofs ofs, uint32_t data_len)\n+{\n+\tstruct icp_qat_fw_la_cipher_req_params *cipher_param;\n+\n+\tcipher_param = (void *)&req->serv_specif_rqpars;\n+\n+\t/* cipher IV */\n+\tset_cipher_iv(cipher_param, iv_vec, ctx->cipher_iv.length, req);\n+\tcipher_param->cipher_offset = ofs.ofs.cipher.head;\n+\tcipher_param->cipher_length = data_len - ofs.ofs.cipher.head;\n+}\n+\n+static __rte_always_inline int\n+qat_sym_dp_submit_single_cipher(void *qp_data, uint8_t *service_data,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs, struct rte_crypto_data *iv_vec,\n+\t__rte_unused struct rte_crypto_data *digest_vec,\n+\t__rte_unused struct rte_crypto_data *aad_vec,\n+\tvoid *opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = service_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = service_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_dp_get_data(qp, req, data, n_data_vecs);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\treq->comn_mid.opaque_data = (uint64_t)opaque;\n+\n+\tsubmit_one_cipher_job(ctx, req, iv_vec, ofs, (uint32_t)data_len);\n+\n+\tservice_ctx->tail = tail;\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline uint32_t\n+qat_sym_dp_submit_cipher_jobs(void *qp_data, uint8_t *service_data,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid **opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = service_ctx->session;\n+\tuint32_t i;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tif (unlikely(qp->enqueued - qp->dequeued + vec->num >=\n+\t\t\tqp->max_inflights)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\treturn 0;\n+\t}\n+\n+\ttail = service_ctx->tail;\n+\n+\tfor (i = 0; i < vec->num; i++) {\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_dp_get_data(qp, req, vec->sgl[i].vec,\n+\t\t\tvec->sgl[i].num) - ofs.ofs.cipher.head -\n+\t\t\tofs.ofs.cipher.tail;\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\t\treq->comn_mid.opaque_data = (uint64_t)opaque[i];\n+\t\tsubmit_one_cipher_job(ctx, req, vec->iv_vec + i, ofs,\n+\t\t\t(uint32_t)data_len);\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\t}\n+\n+\tif (unlikely(i < vec->num))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, vec->num - i);\n+\n+\tservice_ctx->tail = tail;\n+\treturn i;\n+}\n+\n+static __rte_always_inline void\n+submit_one_auth_job(struct qat_sym_session *ctx,\n+\tstruct icp_qat_fw_la_bulk_req *req, struct rte_crypto_data *iv_vec,\n+\tstruct rte_crypto_data *digest_vec, union rte_crypto_sym_ofs ofs,\n+\tuint32_t data_len)\n+{\n+\tstruct icp_qat_fw_la_cipher_req_params *cipher_param;\n+\tstruct icp_qat_fw_la_auth_req_params *auth_param;\n+\n+\tcipher_param = (void *)&req->serv_specif_rqpars;\n+\tauth_param = (void *)((uint8_t *)cipher_param +\n+\t\t\tICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);\n+\n+\tauth_param->auth_off = ofs.ofs.auth.head;\n+\tauth_param->auth_len = data_len - ofs.ofs.auth.head;\n+\tauth_param->auth_res_addr = digest_vec->iova;\n+\n+\tswitch (ctx->qat_hash_alg) {\n+\tcase ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:\n+\tcase ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:\n+\tcase ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:\n+\t\tauth_param->u1.aad_adr = iv_vec->iova;\n+\t\tbreak;\n+\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_128:\n+\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_64:\n+\t\tICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(\n+\t\t\treq->comn_hdr.serv_specif_flags,\n+\t\t\t\tICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);\n+\t\trte_memcpy_generic(cipher_param->u.cipher_IV_array,\n+\t\t\t\tiv_vec->base, ctx->cipher_iv.length);\n+\t\tbreak;\n+\tdefault:\n+\t\tbreak;\n+\t}\n+}\n+\n+static __rte_always_inline int\n+qat_sym_dp_submit_single_auth(void *qp_data, uint8_t *service_data,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs, struct rte_crypto_data *iv_vec,\n+\tstruct rte_crypto_data *digest_vec,\n+\t__rte_unused struct rte_crypto_data *aad_vec,\n+\tvoid *opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = service_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = service_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_dp_get_data(qp, req, data, n_data_vecs);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\treq->comn_mid.opaque_data = (uint64_t)opaque;\n+\n+\tsubmit_one_auth_job(ctx, req, iv_vec, digest_vec, ofs,\n+\t\t\t(uint32_t)data_len);\n+\n+\tservice_ctx->tail = tail;\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline uint32_t\n+qat_sym_dp_submit_auth_jobs(void *qp_data, uint8_t *service_data,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid **opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = service_ctx->session;\n+\tuint32_t i;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tif (unlikely(qp->enqueued - qp->dequeued + vec->num >=\n+\t\t\tqp->max_inflights)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\treturn 0;\n+\t}\n+\n+\ttail = service_ctx->tail;\n+\n+\tfor (i = 0; i < vec->num; i++) {\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_dp_get_data(qp, req, vec->sgl[i].vec,\n+\t\t\tvec->sgl[i].num) - ofs.ofs.cipher.head -\n+\t\t\tofs.ofs.cipher.tail;\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\t\treq->comn_mid.opaque_data = (uint64_t)opaque[i];\n+\t\tsubmit_one_auth_job(ctx, req, vec->iv_vec + i,\n+\t\t\tvec->digest_vec + i, ofs, (uint32_t)data_len);\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\t}\n+\n+\tif (unlikely(i < vec->num))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, vec->num - i);\n+\n+\tservice_ctx->tail = tail;\n+\treturn i;\n+}\n+\n+static __rte_always_inline void\n+submit_one_chain_job(struct qat_sym_session *ctx,\n+\tstruct icp_qat_fw_la_bulk_req *req, struct rte_crypto_vec *data,\n+\tuint16_t n_data_vecs, struct rte_crypto_data *iv_vec,\n+\tstruct rte_crypto_data *digest_vec, union rte_crypto_sym_ofs ofs,\n+\tuint32_t data_len)\n+{\n+\tstruct icp_qat_fw_la_cipher_req_params *cipher_param;\n+\tstruct icp_qat_fw_la_auth_req_params *auth_param;\n+\trte_iova_t auth_iova_end;\n+\tint32_t cipher_len, auth_len;\n+\n+\tcipher_param = (void *)&req->serv_specif_rqpars;\n+\tauth_param = (void *)((uint8_t *)cipher_param +\n+\t\t\tICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);\n+\n+\tcipher_len = data_len - ofs.ofs.cipher.head -\n+\t\t\tofs.ofs.cipher.tail;\n+\tauth_len = data_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail;\n+\n+\tassert(cipher_len > 0 && auth_len > 0);\n+\n+\tcipher_param->cipher_offset = ofs.ofs.cipher.head;\n+\tcipher_param->cipher_length = cipher_len;\n+\tset_cipher_iv(cipher_param, iv_vec, ctx->cipher_iv.length, req);\n+\n+\tauth_param->auth_off = ofs.ofs.cipher.head;\n+\tauth_param->auth_len = auth_len;\n+\tauth_param->auth_res_addr = digest_vec->iova;\n+\n+\tswitch (ctx->qat_hash_alg) {\n+\tcase ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:\n+\tcase ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:\n+\tcase ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:\n+\t\tauth_param->u1.aad_adr = iv_vec->iova;\n+\n+\t\tif (unlikely(n_data_vecs > 1)) {\n+\t\t\tint auth_end_get = 0, i = n_data_vecs - 1;\n+\t\t\tstruct rte_crypto_vec *cvec = &data[i];\n+\t\t\tuint32_t len;\n+\n+\t\t\tlen = data_len - ofs.ofs.auth.tail;\n+\n+\t\t\twhile (i >= 0 && len > 0) {\n+\t\t\t\tif (cvec->len >= len) {\n+\t\t\t\t\tauth_iova_end = cvec->iova +\n+\t\t\t\t\t\t(cvec->len - len);\n+\t\t\t\t\tlen = 0;\n+\t\t\t\t\tauth_end_get = 1;\n+\t\t\t\t\tbreak;\n+\t\t\t\t}\n+\t\t\t\tlen -= cvec->len;\n+\t\t\t\ti--;\n+\t\t\t\tcvec--;\n+\t\t\t}\n+\n+\t\t\tassert(auth_end_get != 0);\n+\t\t} else\n+\t\t\tauth_iova_end = digest_vec->iova +\n+\t\t\t\tctx->digest_length;\n+\n+\t\t/* Then check if digest-encrypted conditions are met */\n+\t\tif ((auth_param->auth_off + auth_param->auth_len <\n+\t\t\t\tcipher_param->cipher_offset +\n+\t\t\t\tcipher_param->cipher_length) &&\n+\t\t\t\t(digest_vec->iova == auth_iova_end)) {\n+\t\t\t/* Handle partial digest encryption */\n+\t\t\tif (cipher_param->cipher_offset +\n+\t\t\t\t\tcipher_param->cipher_length <\n+\t\t\t\t\tauth_param->auth_off +\n+\t\t\t\t\tauth_param->auth_len +\n+\t\t\t\t\tctx->digest_length)\n+\t\t\t\treq->comn_mid.dst_length =\n+\t\t\t\t\treq->comn_mid.src_length =\n+\t\t\t\t\tauth_param->auth_off +\n+\t\t\t\t\tauth_param->auth_len +\n+\t\t\t\t\tctx->digest_length;\n+\t\t\tstruct icp_qat_fw_comn_req_hdr *header =\n+\t\t\t\t&req->comn_hdr;\n+\t\t\tICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(\n+\t\t\t\theader->serv_specif_flags,\n+\t\t\t\tICP_QAT_FW_LA_DIGEST_IN_BUFFER);\n+\t\t}\n+\t\tbreak;\n+\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_128:\n+\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_64:\n+\t\tbreak;\n+\tdefault:\n+\t\tbreak;\n+\t}\n+}\n+\n+static __rte_always_inline int\n+qat_sym_dp_submit_single_chain(void *qp_data, uint8_t *service_data,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs, struct rte_crypto_data *iv_vec,\n+\tstruct rte_crypto_data *digest_vec,\n+\t__rte_unused struct rte_crypto_data *aad_vec,\n+\tvoid *opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = service_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = service_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_dp_get_data(qp, req, data, n_data_vecs);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\treq->comn_mid.opaque_data = (uint64_t)opaque;\n+\n+\tsubmit_one_chain_job(ctx, req, data, n_data_vecs, iv_vec, digest_vec,\n+\t\tofs, (uint32_t)data_len);\n+\n+\tservice_ctx->tail = tail;\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline uint32_t\n+qat_sym_dp_submit_chain_jobs(void *qp_data, uint8_t *service_data,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid **opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = service_ctx->session;\n+\tuint32_t i;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tif (unlikely(qp->enqueued - qp->dequeued + vec->num >=\n+\t\t\tqp->max_inflights)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\treturn 0;\n+\t}\n+\n+\ttail = service_ctx->tail;\n+\n+\tfor (i = 0; i < vec->num; i++) {\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_dp_get_data(qp, req, vec->sgl[i].vec,\n+\t\t\tvec->sgl[i].num) - ofs.ofs.cipher.head -\n+\t\t\tofs.ofs.cipher.tail;\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\t\treq->comn_mid.opaque_data = (uint64_t)opaque[i];\n+\t\tsubmit_one_chain_job(ctx, req, vec->sgl[i].vec, vec->sgl[i].num,\n+\t\t\tvec->iv_vec + i, vec->digest_vec + i, ofs,\n+\t\t\t(uint32_t)data_len);\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\t}\n+\n+\tif (unlikely(i < vec->num))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, vec->num - i);\n+\n+\tservice_ctx->tail = tail;\n+\treturn i;\n+}\n+\n+static __rte_always_inline uint32_t\n+qat_sym_dp_dequeue(void *qp_data, uint8_t *service_data,\n+\trte_cryptodev_get_dequeue_count_t get_dequeue_count,\n+\trte_cryptodev_post_dequeue_t post_dequeue,\n+\tvoid **out_opaque, uint8_t is_opaque_array,\n+\tuint32_t *n_success_jobs)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *rx_queue = &qp->rx_q;\n+\tstruct icp_qat_fw_comn_resp *resp;\n+\tvoid *resp_opaque;\n+\tuint32_t i, n, inflight;\n+\tuint32_t head;\n+\tuint8_t status;\n+\n+\t*n_success_jobs = 0;\n+\thead = service_ctx->head;\n+\n+\tinflight = qp->enqueued - qp->dequeued;\n+\tif (unlikely(inflight == 0))\n+\t\treturn 0;\n+\n+\tresp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +\n+\t\t\thead);\n+\t/* no operation ready */\n+\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n+\t\treturn 0;\n+\n+\tresp_opaque = (void *)(uintptr_t)resp->opaque_data;\n+\t/* get the dequeue count */\n+\tn = get_dequeue_count(resp_opaque);\n+\tif (unlikely(n == 0))\n+\t\treturn 0;\n+\n+\tout_opaque[0] = resp_opaque;\n+\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n+\tpost_dequeue(resp_opaque, 0, status);\n+\t*n_success_jobs += status;\n+\n+\thead = (head + rx_queue->msg_size) & rx_queue->modulo_mask;\n+\n+\t/* we already finished dequeue when n == 1 */\n+\tif (unlikely(n == 1)) {\n+\t\ti = 1;\n+\t\tgoto end_deq;\n+\t}\n+\n+\tif (is_opaque_array) {\n+\t\tfor (i = 1; i < n; i++) {\n+\t\t\tresp = (struct icp_qat_fw_comn_resp *)(\n+\t\t\t\t(uint8_t *)rx_queue->base_addr + head);\n+\t\t\tif (unlikely(*(uint32_t *)resp ==\n+\t\t\t\t\tADF_RING_EMPTY_SIG))\n+\t\t\t\tgoto end_deq;\n+\t\t\tout_opaque[i] = (void *)(uintptr_t)\n+\t\t\t\t\tresp->opaque_data;\n+\t\t\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n+\t\t\t*n_success_jobs += status;\n+\t\t\tpost_dequeue(out_opaque[i], i, status);\n+\t\t\thead = (head + rx_queue->msg_size) &\n+\t\t\t\t\trx_queue->modulo_mask;\n+\t\t}\n+\n+\t\tgoto end_deq;\n+\t}\n+\n+\t/* opaque is not array */\n+\tfor (i = 1; i < n; i++) {\n+\t\tresp = (struct icp_qat_fw_comn_resp *)(\n+\t\t\t(uint8_t *)rx_queue->base_addr + head);\n+\t\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n+\t\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n+\t\t\tgoto end_deq;\n+\t\thead = (head + rx_queue->msg_size) &\n+\t\t\t\trx_queue->modulo_mask;\n+\t\tpost_dequeue(resp_opaque, i, status);\n+\t\t*n_success_jobs += status;\n+\t}\n+\n+end_deq:\n+\tservice_ctx->head = head;\n+\treturn i;\n+}\n+\n+static __rte_always_inline int\n+qat_sym_dp_dequeue_single_job(void *qp_data, uint8_t *service_data,\n+\t\tvoid **out_opaque)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\tstruct qat_queue *rx_queue = &qp->rx_q;\n+\n+\tregister struct icp_qat_fw_comn_resp *resp;\n+\n+\tresp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +\n+\t\t\tservice_ctx->head);\n+\n+\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n+\t\treturn -1;\n+\n+\t*out_opaque = (void *)(uintptr_t)resp->opaque_data;\n+\n+\tservice_ctx->head = (service_ctx->head + rx_queue->msg_size) &\n+\t\t\trx_queue->modulo_mask;\n+\n+\treturn QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n+}\n+\n+static __rte_always_inline void\n+qat_sym_dp_kick_tail(void *qp_data, uint8_t *service_data, uint32_t n)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\n+\tqp->enqueued += n;\n+\tqp->stats.enqueued_count += n;\n+\n+\tassert(service_ctx->tail == ((tx_queue->tail + tx_queue->msg_size * n) &\n+\t\t\ttx_queue->modulo_mask));\n+\n+\ttx_queue->tail = service_ctx->tail;\n+\n+\tWRITE_CSR_RING_TAIL(qp->mmap_bar_addr,\n+\t\t\ttx_queue->hw_bundle_number,\n+\t\t\ttx_queue->hw_queue_number, tx_queue->tail);\n+\ttx_queue->csr_tail = tx_queue->tail;\n+}\n+\n+static __rte_always_inline void\n+qat_sym_dp_update_head(void *qp_data, uint8_t *service_data, uint32_t n)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_queue *rx_queue = &qp->rx_q;\n+\tstruct qat_sym_dp_service_ctx *service_ctx = (void *)service_data;\n+\n+\tassert(service_ctx->head == ((rx_queue->head + rx_queue->msg_size * n) &\n+\t\t\trx_queue->modulo_mask));\n+\n+\trx_queue->head = service_ctx->head;\n+\trx_queue->nb_processed_responses += n;\n+\tqp->dequeued += n;\n+\tqp->stats.dequeued_count += n;\n+\tif (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {\n+\t\tuint32_t old_head, new_head;\n+\t\tuint32_t max_head;\n+\n+\t\told_head = rx_queue->csr_head;\n+\t\tnew_head = rx_queue->head;\n+\t\tmax_head = qp->nb_descriptors * rx_queue->msg_size;\n+\n+\t\t/* write out free descriptors */\n+\t\tvoid *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;\n+\n+\t\tif (new_head < old_head) {\n+\t\t\tmemset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,\n+\t\t\t\t\tmax_head - old_head);\n+\t\t\tmemset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,\n+\t\t\t\t\tnew_head);\n+\t\t} else {\n+\t\t\tmemset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -\n+\t\t\t\t\told_head);\n+\t\t}\n+\t\trx_queue->nb_processed_responses = 0;\n+\t\trx_queue->csr_head = new_head;\n+\n+\t\t/* write current head to CSR */\n+\t\tWRITE_CSR_RING_HEAD(qp->mmap_bar_addr,\n+\t\t\trx_queue->hw_bundle_number, rx_queue->hw_queue_number,\n+\t\t\tnew_head);\n+\t}\n+}\n+\n+int\n+qat_sym_dp_configure_service_ctx(struct rte_cryptodev *dev, uint16_t qp_id,\n+\tstruct rte_crypto_dp_service_ctx *service_ctx,\n+\tenum rte_crypto_dp_service service_type,\n+\tenum rte_crypto_op_sess_type sess_type,\n+\tunion rte_cryptodev_session_ctx session_ctx,\n+\tuint8_t is_update)\n+{\n+\tstruct qat_qp *qp;\n+\tstruct qat_sym_session *ctx;\n+\tstruct qat_sym_dp_service_ctx *dp_ctx;\n+\n+\tif (service_ctx == NULL || session_ctx.crypto_sess == NULL ||\n+\t\t\tsess_type != RTE_CRYPTO_OP_WITH_SESSION)\n+\t\treturn -EINVAL;\n+\n+\tqp = dev->data->queue_pairs[qp_id];\n+\tctx = (struct qat_sym_session *)get_sym_session_private_data(\n+\t\t\tsession_ctx.crypto_sess, qat_sym_driver_id);\n+\tdp_ctx = (struct qat_sym_dp_service_ctx *)\n+\t\t\tservice_ctx->drv_service_data;\n+\n+\tif (!is_update) {\n+\t\tmemset(service_ctx, 0, sizeof(*service_ctx) +\n+\t\t\t\tsizeof(struct qat_sym_dp_service_ctx));\n+\t\tservice_ctx->qp_data = dev->data->queue_pairs[qp_id];\n+\t\tdp_ctx->tail = qp->tx_q.tail;\n+\t\tdp_ctx->head = qp->rx_q.head;\n+\t}\n+\n+\tdp_ctx->session = ctx;\n+\n+\tservice_ctx->submit_done = qat_sym_dp_kick_tail;\n+\tservice_ctx->dequeue_opaque = qat_sym_dp_dequeue;\n+\tservice_ctx->dequeue_single = qat_sym_dp_dequeue_single_job;\n+\tservice_ctx->dequeue_done = qat_sym_dp_update_head;\n+\n+\tif (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||\n+\t\t\tctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {\n+\t\t/* AES-GCM or AES-CCM */\n+\t\tif (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||\n+\t\t\tctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||\n+\t\t\t(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128\n+\t\t\t&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE\n+\t\t\t&& ctx->qat_hash_alg ==\n+\t\t\t\t\tICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {\n+\t\t\tif (service_type != RTE_CRYPTO_DP_SYM_AEAD)\n+\t\t\t\treturn -1;\n+\t\t\tservice_ctx->submit_vec = qat_sym_dp_submit_aead_jobs;\n+\t\t\tservice_ctx->submit_single_job =\n+\t\t\t\t\tqat_sym_dp_submit_single_aead;\n+\t\t} else {\n+\t\t\tif (service_type != RTE_CRYPTO_DP_SYM_CHAIN)\n+\t\t\t\treturn -1;\n+\t\t\tservice_ctx->submit_vec = qat_sym_dp_submit_chain_jobs;\n+\t\t\tservice_ctx->submit_single_job =\n+\t\t\t\t\tqat_sym_dp_submit_single_chain;\n+\t\t}\n+\t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {\n+\t\tif (service_type != RTE_CRYPTO_DP_SYM_AUTH_ONLY)\n+\t\t\treturn -1;\n+\t\tservice_ctx->submit_vec = qat_sym_dp_submit_auth_jobs;\n+\t\tservice_ctx->submit_single_job = qat_sym_dp_submit_single_auth;\n+\t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {\n+\t\tif (service_type != RTE_CRYPTO_DP_SYM_CIPHER_ONLY)\n+\t\t\treturn -1;\n+\t\tservice_ctx->submit_vec = qat_sym_dp_submit_cipher_jobs;\n+\t\tservice_ctx->submit_single_job =\n+\t\t\tqat_sym_dp_submit_single_cipher;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+int\n+qat_sym_get_service_ctx_size(__rte_unused struct rte_cryptodev *dev)\n+{\n+\treturn sizeof(struct qat_sym_dp_service_ctx);\n+}\ndiff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c\nindex 314742f53..bef08c3bc 100644\n--- a/drivers/crypto/qat/qat_sym_pmd.c\n+++ b/drivers/crypto/qat/qat_sym_pmd.c\n@@ -258,7 +258,11 @@ static struct rte_cryptodev_ops crypto_qat_ops = {\n \t\t/* Crypto related operations */\n \t\t.sym_session_get_size\t= qat_sym_session_get_private_size,\n \t\t.sym_session_configure\t= qat_sym_session_configure,\n-\t\t.sym_session_clear\t= qat_sym_session_clear\n+\t\t.sym_session_clear\t= qat_sym_session_clear,\n+\n+\t\t/* Data plane service related operations */\n+\t\t.get_drv_ctx_size = qat_sym_get_service_ctx_size,\n+\t\t.configure_service = qat_sym_dp_configure_service_ctx,\n };\n \n #ifdef RTE_LIBRTE_SECURITY\n@@ -376,7 +380,8 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,\n \t\t\tRTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |\n \t\t\tRTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |\n \t\t\tRTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |\n-\t\t\tRTE_CRYPTODEV_FF_DIGEST_ENCRYPTED;\n+\t\t\tRTE_CRYPTODEV_FF_DIGEST_ENCRYPTED |\n+\t\t\tRTE_CRYPTODEV_FF_DATA_PLANE_SERVICE;\n \n \tif (rte_eal_process_type() != RTE_PROC_PRIMARY)\n \t\treturn 0;\n",
    "prefixes": [
        "v6",
        "2/4"
    ]
}