From patchwork Thu Dec 29 17:12:22 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tomasz Kulasek X-Patchwork-Id: 18694 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [IPv6:::1]) by dpdk.org (Postfix) with ESMTP id 052D56932; Thu, 29 Dec 2016 18:14:10 +0100 (CET) Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id C521D5598 for ; Thu, 29 Dec 2016 18:14:07 +0100 (CET) Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga103.fm.intel.com with ESMTP; 29 Dec 2016 09:12:41 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos; i="5.33,428,1477983600"; d="scan'208"; a="1087895028" Received: from unknown (HELO Sent) ([10.103.102.79]) by fmsmga001.fm.intel.com with SMTP; 29 Dec 2016 09:12:39 -0800 Received: by Sent (sSMTP sendmail emulation); Thu, 29 Dec 2016 18:12:38 +0100 From: Tomasz Kulasek To: dev@dpdk.org Cc: olivier.matz@6wind.com Date: Thu, 29 Dec 2016 18:12:22 +0100 Message-Id: <1483031543-21196-5-git-send-email-tomaszx.kulasek@intel.com> X-Mailer: git-send-email 2.1.4 In-Reply-To: <1483031543-21196-1-git-send-email-tomaszx.kulasek@intel.com> References: <1480698466-17620-1-git-send-email-tomaszx.kulasek@intel.com> <1483031543-21196-1-git-send-email-tomaszx.kulasek@intel.com> Subject: [dpdk-dev] [PATCH v2 4/5] crypto: add sgl support in openssl PMD X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Previous implementation uses EVP_EncryptUpdate() on whole source buffer limiting its usage to the contiguous buffers. This implementation calls EVP_EncryptUpdate() on each segment for cipher operations in continous mode, before finalization allowing to provide chained mbuf as a source. However, libcrypto library expects continuous destination buffer for output of cipher operations and implementation of openssl PMD is limited the same way, requiring contiguous destination mbuf. Signed-off-by: Tomasz Kulasek --- doc/guides/cryptodevs/openssl.rst | 2 +- drivers/crypto/openssl/rte_openssl_pmd.c | 329 +++++++++++++++++++++++------- 2 files changed, 258 insertions(+), 73 deletions(-) diff --git a/doc/guides/cryptodevs/openssl.rst b/doc/guides/cryptodevs/openssl.rst index d2b5906..05e6d67 100644 --- a/doc/guides/cryptodevs/openssl.rst +++ b/doc/guides/cryptodevs/openssl.rst @@ -112,6 +112,6 @@ Limitations ----------- * Maximum number of sessions is 2048. -* Chained mbufs are not supported. +* Chained mbufs are supported only for source mbuf (destination must be contiguous). * Hash only is not supported for GCM and GMAC. * Cipher only is not supported for GCM and GMAC. diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c index 5f8fa33..22639b4 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd.c +++ b/drivers/crypto/openssl/rte_openssl_pmd.c @@ -484,22 +484,110 @@ * Process Operations *------------------------------------------------------------------------------ */ +static inline int +process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, + uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) +{ + struct rte_mbuf *m; + int dstlen; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + return -1; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) + return -1; + *dst += l; + return 0; + } + + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + + *dst += dstlen; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + *dst += dstlen; + n -= l; + } + + return 0; +} + +static inline int +process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, + uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) +{ + struct rte_mbuf *m; + int dstlen; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + return -1; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) + return -1; + *dst += l; + return 0; + } + + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + + *dst += dstlen; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + *dst += dstlen; + n -= l; + } + + return 0; +} /** Process standard openssl cipher encryption */ static int -process_openssl_cipher_encrypt(uint8_t *src, uint8_t *dst, - uint8_t *iv, uint8_t *key, int srclen, +process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, uint8_t *key, int srclen, EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) { - int dstlen, totlen; + int totlen; if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0) goto process_cipher_encrypt_err; - if (EVP_EncryptUpdate(ctx, dst, &dstlen, src, srclen) <= 0) + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx)) goto process_cipher_encrypt_err; - if (EVP_EncryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0) + if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) goto process_cipher_encrypt_err; return 0; @@ -511,11 +599,11 @@ /** Process standard openssl cipher decryption */ static int -process_openssl_cipher_decrypt(uint8_t *src, uint8_t *dst, - uint8_t *iv, uint8_t *key, int srclen, +process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, uint8_t *key, int srclen, EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) { - int dstlen, totlen; + int totlen; if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0) goto process_cipher_decrypt_err; @@ -523,12 +611,12 @@ if (EVP_CIPHER_CTX_set_padding(ctx, 0) <= 0) goto process_cipher_decrypt_err; - if (EVP_DecryptUpdate(ctx, dst, &dstlen, src, srclen) <= 0) + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx)) goto process_cipher_decrypt_err; - if (EVP_DecryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0) + if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) goto process_cipher_decrypt_err; - return 0; process_cipher_decrypt_err: @@ -538,11 +626,25 @@ /** Process cipher des 3 ctr encryption, decryption algorithm */ static int -process_openssl_cipher_des3ctr(uint8_t *src, uint8_t *dst, - uint8_t *iv, uint8_t *key, int srclen, EVP_CIPHER_CTX *ctx) +process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, uint8_t *key, int srclen, + EVP_CIPHER_CTX *ctx) { uint8_t ebuf[8], ctr[8]; int unused, n; + struct rte_mbuf *m; + uint8_t *src; + int l; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_cipher_des3ctr_err; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + l = rte_pktmbuf_data_len(m) - offset; /* We use 3DES encryption also for decryption. * IV is not important for 3DES ecb @@ -551,9 +653,8 @@ goto process_cipher_des3ctr_err; memcpy(ctr, iv, 8); - n = 0; - while (n < srclen) { + for (n = 0; n < srclen; n++) { if (n % 8 == 0) { if (EVP_EncryptUpdate(ctx, (unsigned char *)&ebuf, &unused, @@ -561,8 +662,16 @@ goto process_cipher_des3ctr_err; ctr_inc(ctr); } - dst[n] = src[n] ^ ebuf[n % 8]; - n++; + dst[n] = *(src++) ^ ebuf[n % 8]; + + l--; + if (!l) { + m = m->next; + if (m) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m); + } + } } return 0; @@ -574,9 +683,9 @@ /** Process auth/encription aes-gcm algorithm */ static int -process_openssl_auth_encryption_gcm(uint8_t *src, int srclen, - uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, - uint8_t *key, uint8_t *dst, uint8_t *tag, +process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, + uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) { int len = 0, unused = 0; @@ -591,20 +700,20 @@ if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) goto process_auth_encryption_gcm_err; - if (aadlen > 0) { + if (aadlen > 0) if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) goto process_auth_encryption_gcm_err; - /* Workaround open ssl bug in version less then 1.0.1f */ - if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) - goto process_auth_encryption_gcm_err; - } - if (srclen > 0) - if (EVP_EncryptUpdate(ctx, dst, &len, src, srclen) <= 0) + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx)) goto process_auth_encryption_gcm_err; - if (EVP_EncryptFinal_ex(ctx, dst + len, &len) <= 0) + /* Workaround open ssl bug in version less then 1.0.1f */ + if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) + goto process_auth_encryption_gcm_err; + + if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) goto process_auth_encryption_gcm_err; if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) @@ -618,10 +727,10 @@ } static int -process_openssl_auth_decryption_gcm(uint8_t *src, int srclen, - uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, - uint8_t *key, uint8_t *dst, uint8_t *tag, - EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) +process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, + uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx, + const EVP_CIPHER *algo) { int len = 0, unused = 0; uint8_t empty[] = {}; @@ -638,20 +747,20 @@ if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) goto process_auth_decryption_gcm_err; - if (aadlen > 0) { + if (aadlen > 0) if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) goto process_auth_decryption_gcm_err; - /* Workaround open ssl bug in version less then 1.0.1f */ - if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) - goto process_auth_decryption_gcm_err; - } - if (srclen > 0) - if (EVP_DecryptUpdate(ctx, dst, &len, src, srclen) <= 0) + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx)) goto process_auth_decryption_gcm_err; - if (EVP_DecryptFinal_ex(ctx, dst + len, &len) <= 0) + /* Workaround open ssl bug in version less then 1.0.1f */ + if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) + goto process_auth_decryption_gcm_err; + + if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) goto process_auth_decryption_gcm_final_err; return 0; @@ -666,21 +775,50 @@ /** Process standard openssl auth algorithms */ static int -process_openssl_auth(uint8_t *src, uint8_t *dst, +process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) { size_t dstlen; + struct rte_mbuf *m; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_auth_err; if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) goto process_auth_err; - if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) + goto process_auth_err; + goto process_auth_final; + } + + if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) goto process_auth_err; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) + goto process_auth_err; + n -= l; + } + +process_auth_final: if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) goto process_auth_err; - return 0; process_auth_err: @@ -690,18 +828,48 @@ /** Process standard openssl auth algorithms with hmac */ static int -process_openssl_auth_hmac(uint8_t *src, uint8_t *dst, +process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, __rte_unused uint8_t *iv, EVP_PKEY *pkey, - int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) + int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) { size_t dstlen; + struct rte_mbuf *m; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_auth_err; if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0) goto process_auth_err; - if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) + goto process_auth_err; + goto process_auth_final; + } + + if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) goto process_auth_err; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) + goto process_auth_err; + n -= l; + } + +process_auth_final: if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0) goto process_auth_err; @@ -721,9 +889,18 @@ struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) { /* cipher */ - uint8_t *src = NULL, *dst = NULL, *iv, *tag, *aad; + uint8_t *dst = NULL, *iv, *tag, *aad; int srclen, ivlen, aadlen, status = -1; + /* + * Segmented destination buffer is not supported for + * encryption/decryption + */ + if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return; + } + iv = op->sym->cipher.iv.data; ivlen = op->sym->cipher.iv.length; aad = op->sym->auth.aad.data; @@ -739,22 +916,22 @@ srclen = 0; else { srclen = op->sym->cipher.data.length; - src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, - op->sym->cipher.data.offset); dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, op->sym->cipher.data.offset); } if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) status = process_openssl_auth_encryption_gcm( - src, srclen, aad, aadlen, iv, ivlen, - sess->cipher.key.data, dst, tag, - sess->cipher.ctx, sess->cipher.evp_algo); + mbuf_src, op->sym->cipher.data.offset, srclen, + aad, aadlen, iv, ivlen, sess->cipher.key.data, + dst, tag, sess->cipher.ctx, + sess->cipher.evp_algo); else status = process_openssl_auth_decryption_gcm( - src, srclen, aad, aadlen, iv, ivlen, - sess->cipher.key.data, dst, tag, - sess->cipher.ctx, sess->cipher.evp_algo); + mbuf_src, op->sym->cipher.data.offset, srclen, + aad, aadlen, iv, ivlen, sess->cipher.key.data, + dst, tag, sess->cipher.ctx, + sess->cipher.evp_algo); if (status != 0) { if (status == (-EFAULT) && @@ -772,12 +949,19 @@ (struct rte_crypto_op *op, struct openssl_session *sess, struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) { - uint8_t *src, *dst, *iv; + uint8_t *dst, *iv; int srclen, status; + /* + * Segmented destination buffer is not supported for + * encryption/decryption + */ + if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return; + } + srclen = op->sym->cipher.data.length; - src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, - op->sym->cipher.data.offset); dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, op->sym->cipher.data.offset); @@ -785,17 +969,20 @@ if (sess->cipher.mode == OPENSSL_CIPHER_LIB) if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) - status = process_openssl_cipher_encrypt(src, dst, iv, + status = process_openssl_cipher_encrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, sess->cipher.key.data, srclen, sess->cipher.ctx, sess->cipher.evp_algo); else - status = process_openssl_cipher_decrypt(src, dst, iv, + status = process_openssl_cipher_decrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, sess->cipher.key.data, srclen, sess->cipher.ctx, sess->cipher.evp_algo); else - status = process_openssl_cipher_des3ctr(src, dst, iv, + status = process_openssl_cipher_des3ctr(mbuf_src, dst, + op->sym->cipher.data.offset, iv, sess->cipher.key.data, srclen, sess->cipher.ctx); @@ -809,12 +996,10 @@ (struct rte_crypto_op *op, struct openssl_session *sess, struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) { - uint8_t *src, *dst; + uint8_t *dst; int srclen, status; srclen = op->sym->auth.data.length; - src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, - op->sym->auth.data.offset); if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, @@ -829,13 +1014,14 @@ switch (sess->auth.mode) { case OPENSSL_AUTH_AS_AUTH: - status = process_openssl_auth(src, dst, - NULL, NULL, srclen, + status = process_openssl_auth(mbuf_src, dst, + op->sym->auth.data.offset, NULL, NULL, srclen, sess->auth.auth.ctx, sess->auth.auth.evp_algo); break; case OPENSSL_AUTH_AS_HMAC: - status = process_openssl_auth_hmac(src, dst, - NULL, sess->auth.hmac.pkey, srclen, + status = process_openssl_auth_hmac(mbuf_src, dst, + op->sym->auth.data.offset, NULL, + sess->auth.hmac.pkey, srclen, sess->auth.hmac.ctx, sess->auth.hmac.evp_algo); break; default: @@ -849,8 +1035,7 @@ op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(mbuf_src, - op->sym->auth.digest.length); + rte_pktmbuf_trim(mbuf_src, op->sym->auth.digest.length); } if (status != 0) @@ -901,7 +1086,6 @@ op->sym->session = NULL; } - if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; @@ -995,7 +1179,8 @@ dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | - RTE_CRYPTODEV_FF_CPU_AESNI; + RTE_CRYPTODEV_FF_CPU_AESNI | + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; /* Set vector instructions mode supported */ internals = dev->data->dev_private;