From patchwork Fri Sep 7 10:44:06 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Fan Zhang X-Patchwork-Id: 44389 X-Patchwork-Delegate: gakhil@marvell.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 25B3A4C8E; Fri, 7 Sep 2018 12:58:16 +0200 (CEST) Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by dpdk.org (Postfix) with ESMTP id 1444611A4 for ; Fri, 7 Sep 2018 12:58:12 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 07 Sep 2018 03:58:12 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.53,342,1531810800"; d="scan'208";a="260657028" Received: from silpixa00398673.ir.intel.com (HELO silpixa00398673.ger.corp.intel.com) ([10.237.223.54]) by fmsmga005.fm.intel.com with ESMTP; 07 Sep 2018 03:58:11 -0700 From: Fan Zhang To: dev@dpdk.org Cc: akhil.goyal@nxp.com Date: Fri, 7 Sep 2018 11:44:06 +0100 Message-Id: <20180907104408.36469-2-roy.fan.zhang@intel.com> X-Mailer: git-send-email 2.13.6 In-Reply-To: <20180907104408.36469-1-roy.fan.zhang@intel.com> References: <20180907104408.36469-1-roy.fan.zhang@intel.com> Subject: [dpdk-dev] [PATCH 1/3] crypto/aesni_mb: add aes-gcm algorithm support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch adds AES-GCM algorithm support to AESNI-MB PMD. Signed-off-by: Fan Zhang --- drivers/crypto/aesni_mb/aesni_mb_ops.h | 28 +++- drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c | 159 +++++++++++++++------ drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 30 ++++ drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h | 3 + 4 files changed, 172 insertions(+), 48 deletions(-) diff --git a/drivers/crypto/aesni_mb/aesni_mb_ops.h b/drivers/crypto/aesni_mb/aesni_mb_ops.h index 5a1cba6cb..7257e40aa 100644 --- a/drivers/crypto/aesni_mb/aesni_mb_ops.h +++ b/drivers/crypto/aesni_mb/aesni_mb_ops.h @@ -39,6 +39,8 @@ typedef void (*aes_cmac_sub_key_gen_t) (const void *exp_key, void *k2, void *k3); typedef void (*aes_cmac_keyexp_t) (const void *key, void *keyexp); +typedef void (*aes_gcm_keyexp_t) + (const void *key, struct gcm_key_data *keyexp); /** Multi-buffer library function pointer table */ struct aesni_mb_op_fns { @@ -86,6 +88,12 @@ struct aesni_mb_op_fns { /**< AES CMAC subkey expansions */ aes_cmac_keyexp_t aes_cmac_expkey; /**< AES CMAC key expansions */ + aes_gcm_keyexp_t aes_gcm_128; + /**< AES GCM 128 key expansions */ + aes_gcm_keyexp_t aes_gcm_192; + /**< AES GCM 192 key expansions */ + aes_gcm_keyexp_t aes_gcm_256; + /**< AES GCM 256 key expansions */ } keyexp; /**< Key expansion functions */ } aux; @@ -130,7 +138,10 @@ static const struct aesni_mb_op_fns job_ops[] = { aes_keyexp_256_sse, aes_xcbc_expand_key_sse, aes_cmac_subkey_gen_sse, - aes_keyexp_128_enc_sse + aes_keyexp_128_enc_sse, + aes_gcm_pre_128_sse, + aes_gcm_pre_192_sse, + aes_gcm_pre_256_sse } } }, @@ -157,7 +168,10 @@ static const struct aesni_mb_op_fns job_ops[] = { aes_keyexp_256_avx, aes_xcbc_expand_key_avx, aes_cmac_subkey_gen_avx, - aes_keyexp_128_enc_avx + aes_keyexp_128_enc_avx, + aes_gcm_pre_128_avx_gen2, + aes_gcm_pre_192_avx_gen2, + aes_gcm_pre_256_avx_gen2 } } }, @@ -184,7 +198,10 @@ static const struct aesni_mb_op_fns job_ops[] = { aes_keyexp_256_avx2, aes_xcbc_expand_key_avx2, aes_cmac_subkey_gen_avx2, - aes_keyexp_128_enc_avx2 + aes_keyexp_128_enc_avx2, + aes_gcm_pre_128_avx_gen4, + aes_gcm_pre_192_avx_gen4, + aes_gcm_pre_256_avx_gen4 } } }, @@ -211,7 +228,10 @@ static const struct aesni_mb_op_fns job_ops[] = { aes_keyexp_256_avx512, aes_xcbc_expand_key_avx512, aes_cmac_subkey_gen_avx512, - aes_keyexp_128_enc_avx512 + aes_keyexp_128_enc_avx512, + aes_gcm_pre_128_avx_gen4, + aes_gcm_pre_192_avx_gen4, + aes_gcm_pre_256_avx_gen4 } } } diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 93dc7a443..fe04ae48c 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -83,7 +83,8 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform) } if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { - if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) { + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM || + xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) { if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) return AESNI_MB_OP_AEAD_CIPHER_HASH; else @@ -330,7 +331,10 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { - aes_keyexp_t aes_keyexp_fn; + union { + aes_keyexp_t aes_keyexp_fn; + aes_gcm_keyexp_t aes_gcm_keyexp_fn; + } keyexp; switch (xform->aead.op) { case RTE_CRYPTO_AEAD_OP_ENCRYPT: @@ -350,7 +354,53 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, case RTE_CRYPTO_AEAD_AES_CCM: sess->cipher.mode = CCM; sess->auth.algo = AES_CCM; + + /* Check key length and choose key expansion function for AES */ + switch (xform->aead.key.length) { + case AES_128_BYTES: + sess->cipher.key_length_in_bytes = AES_128_BYTES; + keyexp.aes_keyexp_fn = mb_ops->aux.keyexp.aes128; + break; + default: + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + + /* Expanded cipher keys */ + (*keyexp.aes_keyexp_fn)(xform->aead.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + break; + + case RTE_CRYPTO_AEAD_AES_GCM: + sess->cipher.mode = GCM; + sess->auth.algo = AES_GMAC; + + switch (xform->aead.key.length) { + case AES_128_BYTES: + sess->cipher.key_length_in_bytes = AES_128_BYTES; + keyexp.aes_gcm_keyexp_fn = + mb_ops->aux.keyexp.aes_gcm_128; + break; + case AES_192_BYTES: + sess->cipher.key_length_in_bytes = AES_192_BYTES; + keyexp.aes_gcm_keyexp_fn = + mb_ops->aux.keyexp.aes_gcm_192; + break; + case AES_256_BYTES: + sess->cipher.key_length_in_bytes = AES_256_BYTES; + keyexp.aes_gcm_keyexp_fn = + mb_ops->aux.keyexp.aes_gcm_256; + break; + default: + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + + (keyexp.aes_gcm_keyexp_fn)(xform->aead.key.data, + &sess->cipher.gcm_key); break; + default: AESNI_MB_LOG(ERR, "Unsupported aead mode parameter"); return -ENOTSUP; @@ -360,23 +410,6 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, sess->iv.offset = xform->aead.iv.offset; sess->iv.length = xform->aead.iv.length; - /* Check key length and choose key expansion function for AES */ - - switch (xform->aead.key.length) { - case AES_128_BYTES: - sess->cipher.key_length_in_bytes = AES_128_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes128; - break; - default: - AESNI_MB_LOG(ERR, "Invalid cipher key length"); - return -EINVAL; - } - - /* Expanded cipher keys */ - (*aes_keyexp_fn)(xform->aead.key.data, - sess->cipher.expanded_aes_keys.encode, - sess->cipher.expanded_aes_keys.decode); - return 0; } @@ -573,38 +606,62 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes; - if (job->cipher_mode == DES3) { - job->aes_enc_key_expanded = - session->cipher.exp_3des_keys.ks_ptr; - job->aes_dec_key_expanded = - session->cipher.exp_3des_keys.ks_ptr; - } else { - job->aes_enc_key_expanded = - session->cipher.expanded_aes_keys.encode; - job->aes_dec_key_expanded = - session->cipher.expanded_aes_keys.decode; - } - - - - /* Set authentication parameters */ job->hash_alg = session->auth.algo; - if (job->hash_alg == AES_XCBC) { + + switch (job->hash_alg) { + case AES_XCBC: job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded; job->u.XCBC._k2 = session->auth.xcbc.k2; job->u.XCBC._k3 = session->auth.xcbc.k3; - } else if (job->hash_alg == AES_CCM) { + + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + break; + + case AES_CCM: job->u.CCM.aad = op->sym->aead.aad.data + 18; job->u.CCM.aad_len_in_bytes = session->aead.aad_len; - } else if (job->hash_alg == AES_CMAC) { + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + break; + + case AES_CMAC: job->u.CMAC._key_expanded = session->auth.cmac.expkey; job->u.CMAC._skey1 = session->auth.cmac.skey1; job->u.CMAC._skey2 = session->auth.cmac.skey2; + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + break; - } else { + case AES_GMAC: + job->u.GCM.aad = op->sym->aead.aad.data; + job->u.GCM.aad_len_in_bytes = session->aead.aad_len; + job->aes_enc_key_expanded = &session->cipher.gcm_key; + job->aes_dec_key_expanded = &session->cipher.gcm_key; + break; + + default: job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner; job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer; + + if (job->cipher_mode == DES3) { + job->aes_enc_key_expanded = + session->cipher.exp_3des_keys.ks_ptr; + job->aes_dec_key_expanded = + session->cipher.exp_3des_keys.ks_ptr; + } else { + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + } } /* Mutable crypto operation parameters */ @@ -625,7 +682,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, rte_pktmbuf_data_len(op->sym->m_src)); } else { m_dst = m_src; - if (job->hash_alg == AES_CCM) + if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) m_offset = op->sym->aead.data.offset; else m_offset = op->sym->cipher.data.offset; @@ -637,7 +694,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->auth_tag_output = qp->temp_digests[*digest_idx]; *digest_idx = (*digest_idx + 1) % MAX_JOBS; } else { - if (job->hash_alg == AES_CCM) + if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) job->auth_tag_output = op->sym->aead.digest.data; else job->auth_tag_output = op->sym->auth.digest.data; @@ -647,7 +704,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, * Multi-buffer library current only support returning a truncated * digest length as specified in the relevant IPsec RFCs */ - if (job->hash_alg != AES_CCM && job->hash_alg != AES_CMAC) + if (job->hash_alg != AES_CCM && job->hash_alg != AES_CMAC && + job->hash_alg != AES_GMAC) job->auth_tag_output_len_in_bytes = get_truncated_digest_byte_length(job->hash_alg); else @@ -662,7 +720,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->src = rte_pktmbuf_mtod(m_src, uint8_t *); job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset); - if (job->hash_alg == AES_CCM) { + switch (job->hash_alg) { + case AES_CCM: job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset; job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length; @@ -671,7 +730,19 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset + 1); - } else { + break; + + case AES_GMAC: + job->cipher_start_src_offset_in_bytes = + op->sym->aead.data.offset; + job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset; + job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length; + job->msg_len_to_hash_in_bytes = job->msg_len_to_cipher_in_bytes; + job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); + break; + + default: job->cipher_start_src_offset_in_bytes = op->sym->cipher.data.offset; job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length; @@ -693,7 +764,7 @@ static inline void verify_digest(struct aesni_mb_qp *qp __rte_unused, JOB_AES_HMAC *job, struct rte_crypto_op *op) { /* Verify digest if required */ - if (job->hash_alg == AES_CCM) { + if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) { if (memcmp(job->auth_tag_output, op->sym->aead.digest.data, job->auth_tag_output_len_in_bytes) != 0) op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index ab26e5ae4..5a726bee3 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -330,6 +330,36 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { }, } }, } }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 65535, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + } + }, } + }, } + }, RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h index 70e9d18e5..24a943f2a 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h @@ -91,6 +91,7 @@ static const unsigned auth_digest_byte_lengths[] = { [SHA_512] = 64, [AES_XCBC] = 16, [AES_CMAC] = 16, + [AES_GMAC] = 12, [NULL_HASH] = 0 }; @@ -180,6 +181,8 @@ struct aesni_mb_session { const void *ks_ptr[3]; uint64_t key[3][16]; } exp_3des_keys; + + struct gcm_key_data gcm_key; }; /**< Expanded AES keys - Allocating space to * contain the maximum expanded key size which