Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/93636/?format=api
{ "id": 93636, "url": "https://patches.dpdk.org/api/patches/93636/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20210531141027.13289-14-arkadiuszx.kusztal@intel.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20210531141027.13289-14-arkadiuszx.kusztal@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20210531141027.13289-14-arkadiuszx.kusztal@intel.com", "date": "2021-05-31T14:10:25", "name": "[13/15] crypto/qat: update raw dp api", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "97666081b2176aedaae899a64a82c4dc73edc4a5", "submitter": { "id": 452, "url": "https://patches.dpdk.org/api/people/452/?format=api", "name": "Arkadiusz Kusztal", "email": "arkadiuszx.kusztal@intel.com" }, "delegate": { "id": 6690, "url": "https://patches.dpdk.org/api/users/6690/?format=api", "username": "akhil", "first_name": "akhil", "last_name": "goyal", "email": "gakhil@marvell.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20210531141027.13289-14-arkadiuszx.kusztal@intel.com/mbox/", "series": [ { "id": 17172, "url": "https://patches.dpdk.org/api/series/17172/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=17172", "date": "2021-05-31T14:10:12", "name": "Add support for fourth generation of Intel QuickAssist Technology devices", "version": 1, "mbox": "https://patches.dpdk.org/series/17172/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/93636/comments/", "check": "warning", "checks": "https://patches.dpdk.org/api/patches/93636/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id CEF1DA0524;\n\tMon, 31 May 2021 16:12:29 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id A5C434114C;\n\tMon, 31 May 2021 16:11:10 +0200 (CEST)", "from mga04.intel.com (mga04.intel.com [192.55.52.120])\n by mails.dpdk.org (Postfix) with ESMTP id 91D414114A\n for <dev@dpdk.org>; Mon, 31 May 2021 16:11:09 +0200 (CEST)", "from orsmga004.jf.intel.com ([10.7.209.38])\n by fmsmga104.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 31 May 2021 07:11:09 -0700", "from silpixa00400308.ir.intel.com ([10.237.214.61])\n by orsmga004.jf.intel.com with ESMTP; 31 May 2021 07:11:07 -0700" ], "IronPort-SDR": [ "\n 7bwZ7PJ4k3KEGwjXivbzUH0Fws5uoXuM5hXjg3rA/LAqrwJLuiZCjP3av/Q1eo4TuZ6dyT8uXZ\n n6EVXRQWajWw==", "\n BNb2UvuOBS7MX5BSGnfwYXg9nSalchZ193613u5CuzVWQa8QCrC4PfF7h8zlwmbQpeCH/SlJfw\n CRYtSKU6fOFQ==" ], "X-IronPort-AV": [ "E=McAfee;i=\"6200,9189,10001\"; a=\"201492256\"", "E=Sophos;i=\"5.83,237,1616482800\"; d=\"scan'208\";a=\"201492256\"", "E=Sophos;i=\"5.83,237,1616482800\"; d=\"scan'208\";a=\"548760221\"" ], "X-ExtLoop1": "1", "From": "Arek Kusztal <arkadiuszx.kusztal@intel.com>", "To": "dev@dpdk.org", "Cc": "gakhil@marvell.com,\n\tfiona.trahe@intel.com,\n\troy.fan.zhang@intel.com", "Date": "Mon, 31 May 2021 15:10:25 +0100", "Message-Id": "<20210531141027.13289-14-arkadiuszx.kusztal@intel.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20210531141027.13289-1-arkadiuszx.kusztal@intel.com>", "References": "<20210531141027.13289-1-arkadiuszx.kusztal@intel.com>", "Subject": "[dpdk-dev] [PATCH 13/15] crypto/qat: update raw dp api", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "From: Fan Zhang <roy.fan.zhang@intel.com>\n\nThis commit updates the QAT raw data-path API to support the\nchanges made to device and sessions. The QAT RAW data-path API\nnow works on Generation 1-3 devices.\n\nSigned-off-by: Fan Zhang <roy.fan.zhang@intel.com>\n---\n drivers/crypto/qat/qat_sym_hw_dp.c | 419 +++++++++++++++--------------\n 1 file changed, 216 insertions(+), 203 deletions(-)", "diff": "diff --git a/drivers/crypto/qat/qat_sym_hw_dp.c b/drivers/crypto/qat/qat_sym_hw_dp.c\nindex 2f64de44a1..4305579b54 100644\n--- a/drivers/crypto/qat/qat_sym_hw_dp.c\n+++ b/drivers/crypto/qat/qat_sym_hw_dp.c\n@@ -101,204 +101,6 @@ qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)\n #define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \\\n \tRTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)\n \n-static __rte_always_inline void\n-enqueue_one_aead_job(struct qat_sym_session *ctx,\n-\tstruct icp_qat_fw_la_bulk_req *req,\n-\tstruct rte_crypto_va_iova_ptr *iv,\n-\tstruct rte_crypto_va_iova_ptr *digest,\n-\tstruct rte_crypto_va_iova_ptr *aad,\n-\tunion rte_crypto_sym_ofs ofs, uint32_t data_len)\n-{\n-\tstruct icp_qat_fw_la_cipher_req_params *cipher_param =\n-\t\t(void *)&req->serv_specif_rqpars;\n-\tstruct icp_qat_fw_la_auth_req_params *auth_param =\n-\t\t(void *)((uint8_t *)&req->serv_specif_rqpars +\n-\t\tICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);\n-\tuint8_t *aad_data;\n-\tuint8_t aad_ccm_real_len;\n-\tuint8_t aad_len_field_sz;\n-\tuint32_t msg_len_be;\n-\trte_iova_t aad_iova = 0;\n-\tuint8_t q;\n-\n-\tswitch (ctx->qat_hash_alg) {\n-\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_128:\n-\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_64:\n-\t\tICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(\n-\t\t\treq->comn_hdr.serv_specif_flags,\n-\t\t\t\tICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);\n-\t\trte_memcpy(cipher_param->u.cipher_IV_array, iv->va,\n-\t\t\t\tctx->cipher_iv.length);\n-\t\taad_iova = aad->iova;\n-\t\tbreak;\n-\tcase ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:\n-\t\taad_data = aad->va;\n-\t\taad_iova = aad->iova;\n-\t\taad_ccm_real_len = 0;\n-\t\taad_len_field_sz = 0;\n-\t\tmsg_len_be = rte_bswap32((uint32_t)data_len -\n-\t\t\t\tofs.ofs.cipher.head);\n-\n-\t\tif (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {\n-\t\t\taad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;\n-\t\t\taad_ccm_real_len = ctx->aad_len -\n-\t\t\t\tICP_QAT_HW_CCM_AAD_B0_LEN -\n-\t\t\t\tICP_QAT_HW_CCM_AAD_LEN_INFO;\n-\t\t} else {\n-\t\t\taad_data = iv->va;\n-\t\t\taad_iova = iv->iova;\n-\t\t}\n-\n-\t\tq = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;\n-\t\taad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(\n-\t\t\taad_len_field_sz, ctx->digest_length, q);\n-\t\tif (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {\n-\t\t\tmemcpy(aad_data\t+ ctx->cipher_iv.length +\n-\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET + (q -\n-\t\t\t\tICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),\n-\t\t\t\t(uint8_t *)&msg_len_be,\n-\t\t\t\tICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);\n-\t\t} else {\n-\t\t\tmemcpy(aad_data\t+ ctx->cipher_iv.length +\n-\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n-\t\t\t\t(uint8_t *)&msg_len_be +\n-\t\t\t\t(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE\n-\t\t\t\t- q), q);\n-\t\t}\n-\n-\t\tif (aad_len_field_sz > 0) {\n-\t\t\t*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =\n-\t\t\t\trte_bswap16(aad_ccm_real_len);\n-\n-\t\t\tif ((aad_ccm_real_len + aad_len_field_sz)\n-\t\t\t\t% ICP_QAT_HW_CCM_AAD_B0_LEN) {\n-\t\t\t\tuint8_t pad_len = 0;\n-\t\t\t\tuint8_t pad_idx = 0;\n-\n-\t\t\t\tpad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -\n-\t\t\t\t\t((aad_ccm_real_len +\n-\t\t\t\t\taad_len_field_sz) %\n-\t\t\t\t\tICP_QAT_HW_CCM_AAD_B0_LEN);\n-\t\t\t\tpad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +\n-\t\t\t\t\taad_ccm_real_len +\n-\t\t\t\t\taad_len_field_sz;\n-\t\t\t\tmemset(&aad_data[pad_idx], 0, pad_len);\n-\t\t\t}\n-\t\t}\n-\n-\t\trte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)\n-\t\t\t+ ICP_QAT_HW_CCM_NONCE_OFFSET,\n-\t\t\t(uint8_t *)iv->va +\n-\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);\n-\t\t*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =\n-\t\t\tq - ICP_QAT_HW_CCM_NONCE_OFFSET;\n-\n-\t\trte_memcpy((uint8_t *)aad->va +\n-\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n-\t\t\t(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,\n-\t\t\tctx->cipher_iv.length);\n-\t\tbreak;\n-\tdefault:\n-\t\tbreak;\n-\t}\n-\n-\tcipher_param->cipher_offset = ofs.ofs.cipher.head;\n-\tcipher_param->cipher_length = data_len - ofs.ofs.cipher.head -\n-\t\t\tofs.ofs.cipher.tail;\n-\tauth_param->auth_off = ofs.ofs.cipher.head;\n-\tauth_param->auth_len = cipher_param->cipher_length;\n-\tauth_param->auth_res_addr = digest->iova;\n-\tauth_param->u1.aad_adr = aad_iova;\n-\n-\tif (ctx->is_single_pass) {\n-\t\tcipher_param->spc_aad_addr = aad_iova;\n-\t\tcipher_param->spc_auth_res_addr = digest->iova;\n-\t}\n-}\n-\n-static __rte_always_inline int\n-qat_sym_dp_enqueue_single_aead(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n-\tunion rte_crypto_sym_ofs ofs,\n-\tstruct rte_crypto_va_iova_ptr *iv,\n-\tstruct rte_crypto_va_iova_ptr *digest,\n-\tstruct rte_crypto_va_iova_ptr *aad,\n-\tvoid *user_data)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\tuint32_t tail = dp_ctx->tail;\n-\n-\treq = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n-\tdata_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);\n-\tif (unlikely(data_len < 0))\n-\t\treturn -1;\n-\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;\n-\n-\tenqueue_one_aead_job(ctx, req, iv, digest, aad, ofs,\n-\t\t(uint32_t)data_len);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue++;\n-\n-\treturn 0;\n-}\n-\n-static __rte_always_inline uint32_t\n-qat_sym_dp_enqueue_aead_jobs(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n-\tvoid *user_data[], int *status)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tuint32_t i, n;\n-\tuint32_t tail;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\n-\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n-\tif (unlikely(n == 0)) {\n-\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n-\t\t*status = 0;\n-\t\treturn 0;\n-\t}\n-\n-\ttail = dp_ctx->tail;\n-\n-\tfor (i = 0; i < n; i++) {\n-\t\treq = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\n-\t\tdata_len = qat_sym_dp_parse_data_vec(qp, req, vec->sgl[i].vec,\n-\t\t\tvec->sgl[i].num);\n-\t\tif (unlikely(data_len < 0))\n-\t\t\tbreak;\n-\t\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];\n-\t\tenqueue_one_aead_job(ctx, req, &vec->iv[i], &vec->digest[i],\n-\t\t\t&vec->aad[i], ofs, (uint32_t)data_len);\n-\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\t}\n-\n-\tif (unlikely(i < n))\n-\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue += i;\n-\t*status = 0;\n-\treturn i;\n-}\n-\n static __rte_always_inline void\n enqueue_one_cipher_job(struct qat_sym_session *ctx,\n \tstruct icp_qat_fw_la_bulk_req *req,\n@@ -704,6 +506,207 @@ qat_sym_dp_enqueue_chain_jobs(void *qp_data, uint8_t *drv_ctx,\n \treturn i;\n }\n \n+static __rte_always_inline void\n+enqueue_one_aead_job(struct qat_sym_session *ctx,\n+\tstruct icp_qat_fw_la_bulk_req *req,\n+\tstruct rte_crypto_va_iova_ptr *iv,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *aad,\n+\tunion rte_crypto_sym_ofs ofs, uint32_t data_len)\n+{\n+\tstruct icp_qat_fw_la_cipher_req_params *cipher_param =\n+\t\t(void *)&req->serv_specif_rqpars;\n+\tstruct icp_qat_fw_la_auth_req_params *auth_param =\n+\t\t(void *)((uint8_t *)&req->serv_specif_rqpars +\n+\t\tICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);\n+\tuint8_t *aad_data;\n+\tuint8_t aad_ccm_real_len;\n+\tuint8_t aad_len_field_sz;\n+\tuint32_t msg_len_be;\n+\trte_iova_t aad_iova = 0;\n+\tuint8_t q;\n+\n+\t/* CPM 1.7 uses single pass to treat AEAD as cipher operation */\n+\tif (ctx->is_single_pass) {\n+\t\tenqueue_one_cipher_job(ctx, req, iv, ofs, data_len);\n+\t\tcipher_param->spc_aad_addr = aad->iova;\n+\t\tcipher_param->spc_auth_res_addr = digest->iova;\n+\t\treturn;\n+\t}\n+\n+\tswitch (ctx->qat_hash_alg) {\n+\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_128:\n+\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_64:\n+\t\tICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(\n+\t\t\treq->comn_hdr.serv_specif_flags,\n+\t\t\t\tICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);\n+\t\trte_memcpy(cipher_param->u.cipher_IV_array, iv->va,\n+\t\t\t\tctx->cipher_iv.length);\n+\t\taad_iova = aad->iova;\n+\t\tbreak;\n+\tcase ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:\n+\t\taad_data = aad->va;\n+\t\taad_iova = aad->iova;\n+\t\taad_ccm_real_len = 0;\n+\t\taad_len_field_sz = 0;\n+\t\tmsg_len_be = rte_bswap32((uint32_t)data_len -\n+\t\t\t\tofs.ofs.cipher.head);\n+\n+\t\tif (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {\n+\t\t\taad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;\n+\t\t\taad_ccm_real_len = ctx->aad_len -\n+\t\t\t\tICP_QAT_HW_CCM_AAD_B0_LEN -\n+\t\t\t\tICP_QAT_HW_CCM_AAD_LEN_INFO;\n+\t\t} else {\n+\t\t\taad_data = iv->va;\n+\t\t\taad_iova = iv->iova;\n+\t\t}\n+\n+\t\tq = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;\n+\t\taad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(\n+\t\t\taad_len_field_sz, ctx->digest_length, q);\n+\t\tif (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {\n+\t\t\tmemcpy(aad_data\t+ ctx->cipher_iv.length +\n+\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET + (q -\n+\t\t\t\tICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),\n+\t\t\t\t(uint8_t *)&msg_len_be,\n+\t\t\t\tICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);\n+\t\t} else {\n+\t\t\tmemcpy(aad_data\t+ ctx->cipher_iv.length +\n+\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\t\t(uint8_t *)&msg_len_be +\n+\t\t\t\t(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE\n+\t\t\t\t- q), q);\n+\t\t}\n+\n+\t\tif (aad_len_field_sz > 0) {\n+\t\t\t*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =\n+\t\t\t\trte_bswap16(aad_ccm_real_len);\n+\n+\t\t\tif ((aad_ccm_real_len + aad_len_field_sz)\n+\t\t\t\t% ICP_QAT_HW_CCM_AAD_B0_LEN) {\n+\t\t\t\tuint8_t pad_len = 0;\n+\t\t\t\tuint8_t pad_idx = 0;\n+\n+\t\t\t\tpad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -\n+\t\t\t\t\t((aad_ccm_real_len +\n+\t\t\t\t\taad_len_field_sz) %\n+\t\t\t\t\tICP_QAT_HW_CCM_AAD_B0_LEN);\n+\t\t\t\tpad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +\n+\t\t\t\t\taad_ccm_real_len +\n+\t\t\t\t\taad_len_field_sz;\n+\t\t\t\tmemset(&aad_data[pad_idx], 0, pad_len);\n+\t\t\t}\n+\t\t}\n+\n+\t\trte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)\n+\t\t\t+ ICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\t(uint8_t *)iv->va +\n+\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);\n+\t\t*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =\n+\t\t\tq - ICP_QAT_HW_CCM_NONCE_OFFSET;\n+\n+\t\trte_memcpy((uint8_t *)aad->va +\n+\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\t(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,\n+\t\t\tctx->cipher_iv.length);\n+\t\tbreak;\n+\tdefault:\n+\t\tbreak;\n+\t}\n+\n+\tcipher_param->cipher_offset = ofs.ofs.cipher.head;\n+\tcipher_param->cipher_length = data_len - ofs.ofs.cipher.head -\n+\t\t\tofs.ofs.cipher.tail;\n+\tauth_param->auth_off = ofs.ofs.cipher.head;\n+\tauth_param->auth_len = cipher_param->cipher_length;\n+\tauth_param->auth_res_addr = digest->iova;\n+\tauth_param->u1.aad_adr = aad_iova;\n+}\n+\n+static __rte_always_inline int\n+qat_sym_dp_enqueue_single_aead(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *aad,\n+\tvoid *user_data)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = dp_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;\n+\n+\tenqueue_one_aead_job(ctx, req, iv, digest, aad, ofs,\n+\t\t(uint32_t)data_len);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue++;\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline uint32_t\n+qat_sym_dp_enqueue_aead_jobs(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tuint32_t i, n;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n+\tif (unlikely(n == 0)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\t*status = 0;\n+\t\treturn 0;\n+\t}\n+\n+\ttail = dp_ctx->tail;\n+\n+\tfor (i = 0; i < n; i++) {\n+\t\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_dp_parse_data_vec(qp, req, vec->sgl[i].vec,\n+\t\t\tvec->sgl[i].num);\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\t\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];\n+\t\tenqueue_one_aead_job(ctx, req, &vec->iv[i], &vec->digest[i],\n+\t\t\t&vec->aad[i], ofs, (uint32_t)data_len);\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\t}\n+\n+\tif (unlikely(i < n))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue += i;\n+\t*status = 0;\n+\treturn i;\n+}\n+\n static __rte_always_inline uint32_t\n qat_sym_dp_dequeue_burst(void *qp_data, uint8_t *drv_ctx,\n \trte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,\n@@ -937,8 +940,9 @@ qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,\n \traw_dp_ctx->dequeue = qat_sym_dp_dequeue;\n \traw_dp_ctx->dequeue_done = qat_sym_dp_update_head;\n \n-\tif (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||\n-\t\t\tctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {\n+\tif ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||\n+\t\t\tctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&\n+\t\t\t!ctx->is_gmac) {\n \t\t/* AES-GCM or AES-CCM */\n \t\tif (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||\n \t\t\tctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||\n@@ -954,12 +958,21 @@ qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,\n \t\t\t\t\tqat_sym_dp_enqueue_chain_jobs;\n \t\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_chain;\n \t\t}\n-\t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {\n+\t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {\n \t\traw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs;\n \t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth;\n \t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {\n-\t\traw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_cipher_jobs;\n-\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_cipher;\n+\t\tif (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||\n+\t\t\tctx->qat_cipher_alg ==\n+\t\t\t\tICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {\n+\t\t\traw_dp_ctx->enqueue_burst =\n+\t\t\t\t\tqat_sym_dp_enqueue_aead_jobs;\n+\t\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;\n+\t\t} else {\n+\t\t\traw_dp_ctx->enqueue_burst =\n+\t\t\t\t\tqat_sym_dp_enqueue_cipher_jobs;\n+\t\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_cipher;\n+\t\t}\n \t} else\n \t\treturn -1;\n \n", "prefixes": [ "13/15" ] }