From patchwork Fri Mar 5 12:45:12 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Dybkowski, AdamX" X-Patchwork-Id: 88517 Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4906EA0547; Fri, 5 Mar 2021 13:45:24 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id CA1CE40691; Fri, 5 Mar 2021 13:45:23 +0100 (CET) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by mails.dpdk.org (Postfix) with ESMTP id C17A540147 for ; Fri, 5 Mar 2021 13:45:22 +0100 (CET) IronPort-SDR: S7aDmiBT7PPGd9qS3DBlZscf2giMIc0S22k+Uifq2Z2VxbZ858dDPI2TX+urztlSDVGuu1u7jK tOpXPrq8co7g== X-IronPort-AV: E=McAfee;i="6000,8403,9913"; a="272648806" X-IronPort-AV: E=Sophos;i="5.81,224,1610438400"; d="scan'208";a="272648806" Received: from fmsmga008.fm.intel.com ([10.253.24.58]) by fmsmga105.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 05 Mar 2021 04:45:21 -0800 IronPort-SDR: Lj5iJ4skKvOPd1BsAhffmvAak1SAs7gRr1FyNNaw+8vEdwcNt0N07eVORUSC8OxDT7uIWE0F6I IX8SGSq9+2MQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.81,224,1610438400"; d="scan'208";a="401666743" Received: from silpixa00400308.ir.intel.com ([10.237.214.143]) by fmsmga008.fm.intel.com with ESMTP; 05 Mar 2021 04:45:19 -0800 From: Adam Dybkowski To: dev@dpdk.org, declan.doherty@intel.com, arkadiuszx.kusztal@intel.com Cc: Adam Dybkowski Date: Fri, 5 Mar 2021 12:45:12 +0000 Message-Id: <20210305124512.16286-1-adamx.dybkowski@intel.com> X-Mailer: git-send-email 2.17.1 Subject: [dpdk-dev] [PATCH] crypto/qat: support Single-Pass GMAC on QAT GEN3 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch implements Single-Pass AES-GMAC possible on QAT GEN3 which improves the performance. On GEN1 and GEN2 the previous chained method is used. Signed-off-by: Adam Dybkowski --- drivers/common/qat/qat_qp.c | 3 +- drivers/crypto/qat/qat_sym.c | 64 +++++++++++++++++++++++++- drivers/crypto/qat/qat_sym.h | 67 ++++++++++++++++++---------- drivers/crypto/qat/qat_sym_pmd.c | 6 +++ drivers/crypto/qat/qat_sym_session.c | 48 +++++++++++++++++++- drivers/crypto/qat/qat_sym_session.h | 3 +- 6 files changed, 162 insertions(+), 29 deletions(-) diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c index 32d740105..4a8078541 100644 --- a/drivers/common/qat/qat_qp.c +++ b/drivers/common/qat/qat_qp.c @@ -862,7 +862,8 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops) nb_fw_responses = 1; if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) - qat_sym_process_response(ops, resp_msg); + qat_sym_process_response(ops, resp_msg, + tmp_qp->op_cookies[head >> rx_queue->trailz]); else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) nb_fw_responses = qat_comp_process_response( ops, resp_msg, diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c index 4b7676deb..0363d78b3 100644 --- a/drivers/crypto/qat/qat_sym.c +++ b/drivers/crypto/qat/qat_sym.c @@ -144,6 +144,57 @@ set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset, iv_length); } +/** Handle Single-Pass AES-GMAC on QAT GEN3 */ +static inline void +handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op, + struct qat_sym_op_cookie *cookie, + struct icp_qat_fw_la_bulk_req *qat_req) +{ + static const uint32_t ver_key_offset = + sizeof(struct icp_qat_hw_auth_setup) + + ICP_QAT_HW_GALOIS_128_STATE1_SZ + + ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ + + ICP_QAT_HW_GALOIS_E_CTR0_SZ + + sizeof(struct icp_qat_hw_cipher_config); + struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = + (void *) &qat_req->cd_ctrl; + struct icp_qat_fw_la_cipher_req_params *cipher_param = + (void *) &qat_req->serv_specif_rqpars; + uint32_t data_length = op->sym->auth.data.length; + + /* Fill separate Content Descriptor for this op */ + rte_memcpy(cookie->opt.spc_gmac.cd_cipher.key, + ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ? + ctx->cd.cipher.key : + (const uint8_t *) &ctx->cd + ver_key_offset, + cipher_cd_ctrl->cipher_key_sz << 3); + cookie->opt.spc_gmac.cd_cipher.cipher_config.val = + ICP_QAT_HW_CIPHER_CONFIG_BUILD( + ICP_QAT_HW_CIPHER_AEAD_MODE, + ctx->qat_cipher_alg, + ICP_QAT_HW_CIPHER_NO_CONVERT, + (ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ? + ICP_QAT_HW_CIPHER_ENCRYPT : + ICP_QAT_HW_CIPHER_DECRYPT)); + QAT_FIELD_SET(cookie->opt.spc_gmac.cd_cipher.cipher_config.val, + ctx->digest_length, + QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS, + QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK); + cookie->opt.spc_gmac.cd_cipher.cipher_config.reserved = + ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(data_length); + cookie->opt.spc_gmac.key_size = cipher_cd_ctrl->cipher_key_sz << 3; + + /* Update the request */ + qat_req->cd_pars.u.s.content_desc_addr = + cookie->opt.spc_gmac.cd_phys_addr; + qat_req->comn_mid.src_length = data_length; + qat_req->comn_mid.dst_length = 0; + cipher_param->spc_aad_addr = 0; + cipher_param->spc_auth_res_addr = op->sym->auth.digest.phys_addr; + cipher_param->spc_aad_sz = data_length; + cipher_param->reserved = 0; +} + int qat_sym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie, enum qat_device_gen qat_dev_gen) @@ -161,6 +212,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, uint64_t auth_data_end = 0; uint8_t do_sgl = 0; uint8_t in_place = 1; + uint8_t single_pass_gmac = 0; int alignment_adjustment = 0; struct rte_crypto_op *op = (struct rte_crypto_op *)in_op; struct qat_sym_op_cookie *cookie = @@ -216,7 +268,12 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, } qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; - rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); + if (ctx->is_single_pass_gmac && + op->sym->auth.data.length <= QAT_AES_GMAC_SPC_MAX_SIZE) { + single_pass_gmac = 1; + rte_mov128((uint8_t *)qat_req, (const uint8_t *)&ctx->fw_req2); + } else + rte_mov128((uint8_t *)qat_req, (const uint8_t *)&ctx->fw_req); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; cipher_param = (void *)&qat_req->serv_specif_rqpars; auth_param = (void *)((uint8_t *)cipher_param + @@ -619,11 +676,14 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, qat_req->comn_mid.dest_data_addr = dst_buf_start; } - /* Handle Single-Pass GCM */ if (ctx->is_single_pass) { + /* Handle Single-Pass GCM */ cipher_param->spc_aad_addr = op->sym->aead.aad.phys_addr; cipher_param->spc_auth_res_addr = op->sym->aead.digest.phys_addr; + } else if (single_pass_gmac) { + /* Handle Single-Pass GMAC */ + handle_spc_gmac(ctx, op, cookie, qat_req); } #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h index b58be1282..d3576b470 100644 --- a/drivers/crypto/qat/qat_sym.h +++ b/drivers/crypto/qat/qat_sym.h @@ -29,6 +29,9 @@ */ #define QAT_SYM_SGL_MAX_NUMBER 16 +/* Maximum data length for single pass GMAC: 2^14-1 */ +#define QAT_AES_GMAC_SPC_MAX_SIZE 16383 + struct qat_sym_session; struct qat_sym_sgl { @@ -41,6 +44,15 @@ struct qat_sym_op_cookie { struct qat_sym_sgl qat_sgl_dst; phys_addr_t qat_sgl_src_phys_addr; phys_addr_t qat_sgl_dst_phys_addr; + union { + /* Used for Single-Pass AES-GMAC only */ + struct { + struct icp_qat_hw_cipher_algo_blk cd_cipher + __rte_packed __rte_cache_aligned; + phys_addr_t cd_phys_addr; + uint8_t key_size; + } spc_gmac; + } opt; }; int @@ -212,46 +224,46 @@ qat_sym_preprocess_requests(void **ops __rte_unused, #endif static inline void -qat_sym_process_response(void **op, uint8_t *resp) +qat_sym_process_response(void **op, uint8_t *resp, void *op_cookie) { struct icp_qat_fw_comn_resp *resp_msg = (struct icp_qat_fw_comn_resp *)resp; struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t) (resp_msg->opaque_data); struct qat_sym_session *sess; + uint8_t is_docsis_sec; #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG QAT_DP_HEXDUMP_LOG(DEBUG, "qat_response:", (uint8_t *)resp_msg, sizeof(struct icp_qat_fw_comn_resp)); #endif +#ifdef RTE_LIB_SECURITY + if (rx_op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { + /* + * Assuming at this point that if it's a security + * op, that this is for DOCSIS + */ + sess = (struct qat_sym_session *) + get_sec_session_private_data( + rx_op->sym->sec_session); + is_docsis_sec = 1; + } else +#endif + { + sess = (struct qat_sym_session *) + get_sym_session_private_data( + rx_op->sym->session, + qat_sym_driver_id); + is_docsis_sec = 0; + } + if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( resp_msg->comn_hdr.comn_status)) { rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { -#ifdef RTE_LIB_SECURITY - uint8_t is_docsis_sec = 0; - - if (rx_op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { - /* - * Assuming at this point that if it's a security - * op, that this is for DOCSIS - */ - sess = (struct qat_sym_session *) - get_sec_session_private_data( - rx_op->sym->sec_session); - is_docsis_sec = 1; - } else -#endif - { - sess = (struct qat_sym_session *) - get_sym_session_private_data( - rx_op->sym->session, - qat_sym_driver_id); - } - rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; if (sess->bpi_ctx) { @@ -262,6 +274,14 @@ qat_sym_process_response(void **op, uint8_t *resp) #endif } } + + if (sess->is_single_pass_gmac) { + struct qat_sym_op_cookie *cookie = + (struct qat_sym_op_cookie *) op_cookie; + bzero(cookie->opt.spc_gmac.cd_cipher.key, + cookie->opt.spc_gmac.key_size); + } + *op = (void *)rx_op; } @@ -283,7 +303,8 @@ qat_sym_preprocess_requests(void **ops __rte_unused, } static inline void -qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused) +qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused, + void *op_cookie) { } diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c index 93666fdad..b9601c6c3 100644 --- a/drivers/crypto/qat/qat_sym_pmd.c +++ b/drivers/crypto/qat/qat_sym_pmd.c @@ -211,6 +211,12 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, rte_mempool_virt2iova(cookie) + offsetof(struct qat_sym_op_cookie, qat_sgl_dst); + + cookie->opt.spc_gmac.cd_phys_addr = + rte_mempool_virt2iova(cookie) + + offsetof(struct qat_sym_op_cookie, + opt.spc_gmac.cd_cipher); + } /* Get fw version from QAT (GEN2), skip if we've got it already */ diff --git a/drivers/crypto/qat/qat_sym_session.c b/drivers/crypto/qat/qat_sym_session.c index 23d059bf8..412bef22c 100644 --- a/drivers/crypto/qat/qat_sym_session.c +++ b/drivers/crypto/qat/qat_sym_session.c @@ -532,11 +532,49 @@ qat_sym_session_handle_mixed(const struct rte_cryptodev *dev, } } +static void +qat_sym_session_handle_single_pass_gmac(struct qat_sym_session *session, + struct rte_crypto_auth_xform *auth_xform) +{ + struct icp_qat_fw_la_cipher_req_params *cipher_param = + (void *) &session->fw_req2.serv_specif_rqpars; + struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = + (void *) &session->fw_req2.cd_ctrl; + + session->is_single_pass_gmac = 1; + session->min_qat_dev_gen = QAT_GEN3; + rte_memcpy(&session->fw_req2, &session->fw_req, + sizeof(struct icp_qat_fw_la_bulk_req)); + bzero(&cipher_param->spc_aad_addr, 28); + bzero(&cipher_cd_ctrl->resrvd1, 15); + session->fw_req2.comn_hdr.service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; + session->fw_req2.cd_pars.u.s.content_desc_params_sz = RTE_ALIGN_CEIL( + sizeof(struct icp_qat_hw_cipher_config) + + auth_xform->key.length, 8) >> 3; + cipher_cd_ctrl->cipher_cfg_offset = 0; + ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + session->fw_req2.comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( + session->fw_req2.comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_SINGLE_PASS_PROTO); + ICP_QAT_FW_LA_PROTO_SET( + session->fw_req2.comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_NO_PROTO); + cipher_param->cipher_offset = 0; + cipher_param->cipher_length = 0; + cipher_param->spc_auth_res_sz = auth_xform->digest_length; +} + int qat_sym_session_set_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_sym_session *session = session_private; + struct qat_sym_dev_private *internals = dev->data->dev_private; + enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen; int ret; int qat_cmd_id; @@ -571,6 +609,13 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev, ret = qat_sym_session_configure_auth(dev, xform, session); if (ret < 0) return ret; + session->is_single_pass_gmac = + qat_dev_gen == QAT_GEN3 && + xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC && + xform->auth.iv.length == QAT_AES_GCM_SPC_IV_SIZE; + if (session->is_single_pass_gmac) + qat_sym_session_handle_single_pass_gmac(session, + &xform->auth); break; case ICP_QAT_FW_LA_CMD_CIPHER_HASH: if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { @@ -706,8 +751,8 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, struct qat_sym_dev_private *internals = dev->data->dev_private; const uint8_t *key_data = auth_xform->key.data; uint8_t key_length = auth_xform->key.length; - session->aes_cmac = 0; + session->aes_cmac = 0; session->auth_iv.offset = auth_xform->iv.offset; session->auth_iv.length = auth_xform->iv.length; session->auth_mode = ICP_QAT_HW_AUTH_MODE1; @@ -765,7 +810,6 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; if (session->auth_iv.length == 0) session->auth_iv.length = AES_GCM_J0_LEN; - break; case RTE_CRYPTO_AUTH_SNOW3G_UIA2: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2; diff --git a/drivers/crypto/qat/qat_sym_session.h b/drivers/crypto/qat/qat_sym_session.h index 011e5bb7a..6e1b3d8e7 100644 --- a/drivers/crypto/qat/qat_sym_session.h +++ b/drivers/crypto/qat/qat_sym_session.h @@ -36,7 +36,6 @@ /* 96-bit case of IV for CCP/GCM single pass algorithm */ #define QAT_AES_GCM_SPC_IV_SIZE 12 - #define QAT_AES_HW_CONFIG_CBC_ENC(alg) \ ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ ICP_QAT_HW_CIPHER_NO_CONVERT, \ @@ -76,6 +75,7 @@ struct qat_sym_session { uint8_t *cd_cur_ptr; phys_addr_t cd_paddr; struct icp_qat_fw_la_bulk_req fw_req; + struct icp_qat_fw_la_bulk_req fw_req2; uint8_t aad_len; struct qat_crypto_instance *inst; struct { @@ -91,6 +91,7 @@ struct qat_sym_session { enum qat_device_gen min_qat_dev_gen; uint8_t aes_cmac; uint8_t is_single_pass; + uint8_t is_single_pass_gmac; }; int