Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 49142,
    "url": "http://patches.dpdk.org/api/patches/49142/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/20181219201609.37934-3-roy.fan.zhang@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20181219201609.37934-3-roy.fan.zhang@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20181219201609.37934-3-roy.fan.zhang@intel.com",
    "date": "2018-12-19T20:16:07",
    "name": "[v3,2/4] crypto/aesni_mb: use architecture independent macros",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "6a106c6df01350e6d6136813696cc4237bd37da4",
    "submitter": {
        "id": 304,
        "url": "http://patches.dpdk.org/api/people/304/?format=api",
        "name": "Fan Zhang",
        "email": "roy.fan.zhang@intel.com"
    },
    "delegate": {
        "id": 6690,
        "url": "http://patches.dpdk.org/api/users/6690/?format=api",
        "username": "akhil",
        "first_name": "akhil",
        "last_name": "goyal",
        "email": "gakhil@marvell.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/20181219201609.37934-3-roy.fan.zhang@intel.com/mbox/",
    "series": [
        {
            "id": 2880,
            "url": "http://patches.dpdk.org/api/series/2880/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=2880",
            "date": "2018-12-19T20:16:05",
            "name": "use architecure independent macros",
            "version": 3,
            "mbox": "http://patches.dpdk.org/series/2880/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/49142/comments/",
    "check": "warning",
    "checks": "http://patches.dpdk.org/api/patches/49142/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@dpdk.org",
        "Delivered-To": "patchwork@dpdk.org",
        "Received": [
            "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id BC4A81B5EF;\n\tWed, 19 Dec 2018 21:16:18 +0100 (CET)",
            "from mga17.intel.com (mga17.intel.com [192.55.52.151])\n\tby dpdk.org (Postfix) with ESMTP id F0A941B5A7\n\tfor <dev@dpdk.org>; Wed, 19 Dec 2018 21:16:14 +0100 (CET)",
            "from fmsmga002.fm.intel.com ([10.253.24.26])\n\tby fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t19 Dec 2018 12:16:14 -0800",
            "from silpixa00398673.ir.intel.com (HELO\n\tsilpixa00398673.ger.corp.intel.com) ([10.237.223.54])\n\tby fmsmga002.fm.intel.com with ESMTP; 19 Dec 2018 12:16:12 -0800"
        ],
        "X-Amp-Result": "SKIPPED(no attachment in message)",
        "X-Amp-File-Uploaded": "False",
        "X-ExtLoop1": "1",
        "X-IronPort-AV": "E=Sophos;i=\"5.56,374,1539673200\"; d=\"scan'208\";a=\"127435143\"",
        "From": "Fan Zhang <roy.fan.zhang@intel.com>",
        "To": "dev@dpdk.org",
        "Cc": "akhil.goyal@nxp.com, thomas@monjalon.net,\n\tLukasz Krakowiak <lukaszx.krakowiak@intel.com>",
        "Date": "Wed, 19 Dec 2018 20:16:07 +0000",
        "Message-Id": "<20181219201609.37934-3-roy.fan.zhang@intel.com>",
        "X-Mailer": "git-send-email 2.13.6",
        "In-Reply-To": "<20181219201609.37934-1-roy.fan.zhang@intel.com>",
        "References": "<20181211122917.18713-1-roy.fan.zhang@intel.com>\n\t<20181219201609.37934-1-roy.fan.zhang@intel.com>",
        "Subject": "[dpdk-dev] [PATCH v3 2/4] crypto/aesni_mb: use architecture\n\tindependent macros",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.15",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "This patch duplicates the original rte_aesni_mb_pmd*.c files and replaces\nthe function calls provided by intel-ipsec-mb library into\narchitecture-independent macros. The build systems are updated to choose\ncompiling either rte_aesni_mb_pmd*.c or rte_aesni_mb_pmd*_compat.c based\non the installed intel-ipsec-mb version. For the intel-ipsec-mb older\nthan 0.52.0 rte_aesni_mb_pmd*_compat.c will be compiled, otherwise\nrte_aesni_mb_pmd*.c will be compiled.\n\nSigned-off-by: Fan Zhang <roy.fan.zhang@intel.com>\nSigned-off-by: Lukasz Krakowiak <lukaszx.krakowiak@intel.com>\nAcked-by: Damian Nowak <damianx.nowak@intel.com>\n---\n drivers/crypto/aesni_mb/Makefile                   |   26 +-\n drivers/crypto/aesni_mb/meson.build                |   18 +-\n drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c         | 1237 ++++++++++++++++++++\n drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c     |  681 +++++++++++\n drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h |   52 +-\n 5 files changed, 2003 insertions(+), 11 deletions(-)\n create mode 100644 drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c\n create mode 100644 drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c",
    "diff": "diff --git a/drivers/crypto/aesni_mb/Makefile b/drivers/crypto/aesni_mb/Makefile\nindex 5a8671cd4..c2bda5838 100644\n--- a/drivers/crypto/aesni_mb/Makefile\n+++ b/drivers/crypto/aesni_mb/Makefile\n@@ -1,5 +1,5 @@\n # SPDX-License-Identifier: BSD-3-Clause\n-# Copyright(c) 2015 Intel Corporation\n+# Copyright(c) 2015-2018 Intel Corporation\n \n include $(RTE_SDK)/mk/rte.vars.mk\n \n@@ -22,8 +22,26 @@ LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring\n LDLIBS += -lrte_cryptodev\n LDLIBS += -lrte_bus_vdev\n \n-# library source files\n-SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd_compat.c\n-SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd_ops_compat.c\n+IMB_HDR = /usr/include/intel-ipsec-mb.h\n+\n+# Detect library version\n+IMB_VERSION = $(shell grep -e \"IMB_VERSION_STR\" $(IMB_HDR) | cut -d'\"' -f2)\n+IMB_VERSION_NUM = $(shell grep -e \"IMB_VERSION_NUM\" $(IMB_HDR) | cut -d' ' -f3)\n+\n+ifeq ($(IMB_VERSION),)\n+\t# files for older version of IMB\n+\tSRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd_compat.c\n+\tSRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd_ops_compat.c\n+else\n+\tifeq ($(shell expr $(IMB_VERSION_NUM) \\>= 0x3400), 1)\n+\t\t# files for a new version of IMB\n+\t\tSRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd.c\n+\t\tSRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd_ops.c\n+\telse\n+\t\t# files for older version of IMB\n+\t\tSRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd_compat.c\n+\t\tSRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd_ops_compat.c\n+\tendif\n+endif\n \n include $(RTE_SDK)/mk/rte.lib.mk\ndiff --git a/drivers/crypto/aesni_mb/meson.build b/drivers/crypto/aesni_mb/meson.build\nindex ed68c7f39..b5a7fa7f7 100644\n--- a/drivers/crypto/aesni_mb/meson.build\n+++ b/drivers/crypto/aesni_mb/meson.build\n@@ -1,6 +1,6 @@\n # SPDX-License-Identifier: BSD-3-Clause\n # Copyright(c) 2018 Intel Corporation\n-\n+IPSec_MB_ver_0_52 = '0.52.0'\n lib = cc.find_library('IPSec_MB', required: false)\n if not lib.found()\n \tbuild = false\n@@ -8,5 +8,19 @@ else\n \text_deps += lib\n endif\n \n-sources = files('rte_aesni_mb_pmd_compat.c', 'rte_aesni_mb_pmd_ops_compat.c')\n+imb_arr = cc.get_define('IMB_VERSION_STR',\n+\tprefix : '#include<intel-ipsec-mb.h>').split('\"')\n+\n+imb_ver =''.join(imb_arr)\n+\n+if imb_ver.version_compare('>' + IPSec_MB_ver_0_52)\n+\tmessage('Build for a new version of library IPSec_MB[' + imb_ver + ']')\n+\tsources = files('rte_aesni_mb_pmd.c',\n+\t\t'rte_aesni_mb_pmd_ops.c')\n+else\n+\tsources = files('rte_aesni_mb_pmd_compat.c',\n+\t\t'rte_aesni_mb_pmd_ops_compat.c')\n+\tmessage('Build for older version of library IPSec_MB[' + imb_ver + ']')\n+endif\n+\n deps += ['bus_vdev']\ndiff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c\nnew file mode 100644\nindex 000000000..2c25b7b32\n--- /dev/null\n+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c\n@@ -0,0 +1,1237 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(c) 2015-2017 Intel Corporation\n+ */\n+\n+#include <intel-ipsec-mb.h>\n+\n+#include <rte_common.h>\n+#include <rte_hexdump.h>\n+#include <rte_cryptodev.h>\n+#include <rte_cryptodev_pmd.h>\n+#include <rte_bus_vdev.h>\n+#include <rte_malloc.h>\n+#include <rte_cpuflags.h>\n+\n+#include \"rte_aesni_mb_pmd_private.h\"\n+\n+#define AES_CCM_DIGEST_MIN_LEN 4\n+#define AES_CCM_DIGEST_MAX_LEN 16\n+#define HMAC_MAX_BLOCK_SIZE 128\n+static uint8_t cryptodev_driver_id;\n+\n+typedef void (*hash_one_block_t)(const void *data, void *digest);\n+typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);\n+\n+/**\n+ * Calculate the authentication pre-computes\n+ *\n+ * @param one_block_hash\tFunction pointer to calculate digest on ipad/opad\n+ * @param ipad\t\t\tInner pad output byte array\n+ * @param opad\t\t\tOuter pad output byte array\n+ * @param hkey\t\t\tAuthentication key\n+ * @param hkey_len\t\tAuthentication key length\n+ * @param blocksize\t\tBlock size of selected hash algo\n+ */\n+static void\n+calculate_auth_precomputes(hash_one_block_t one_block_hash,\n+\t\tuint8_t *ipad, uint8_t *opad,\n+\t\tuint8_t *hkey, uint16_t hkey_len,\n+\t\tuint16_t blocksize)\n+{\n+\tunsigned i, length;\n+\n+\tuint8_t ipad_buf[blocksize] __rte_aligned(16);\n+\tuint8_t opad_buf[blocksize] __rte_aligned(16);\n+\n+\t/* Setup inner and outer pads */\n+\tmemset(ipad_buf, HMAC_IPAD_VALUE, blocksize);\n+\tmemset(opad_buf, HMAC_OPAD_VALUE, blocksize);\n+\n+\t/* XOR hash key with inner and outer pads */\n+\tlength = hkey_len > blocksize ? blocksize : hkey_len;\n+\n+\tfor (i = 0; i < length; i++) {\n+\t\tipad_buf[i] ^= hkey[i];\n+\t\topad_buf[i] ^= hkey[i];\n+\t}\n+\n+\t/* Compute partial hashes */\n+\t(*one_block_hash)(ipad_buf, ipad);\n+\t(*one_block_hash)(opad_buf, opad);\n+\n+\t/* Clean up stack */\n+\tmemset(ipad_buf, 0, blocksize);\n+\tmemset(opad_buf, 0, blocksize);\n+}\n+\n+/** Get xform chain order */\n+static enum aesni_mb_operation\n+aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)\n+{\n+\tif (xform == NULL)\n+\t\treturn AESNI_MB_OP_NOT_SUPPORTED;\n+\n+\tif (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {\n+\t\tif (xform->next == NULL)\n+\t\t\treturn AESNI_MB_OP_CIPHER_ONLY;\n+\t\tif (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)\n+\t\t\treturn AESNI_MB_OP_CIPHER_HASH;\n+\t}\n+\n+\tif (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {\n+\t\tif (xform->next == NULL)\n+\t\t\treturn AESNI_MB_OP_HASH_ONLY;\n+\t\tif (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)\n+\t\t\treturn AESNI_MB_OP_HASH_CIPHER;\n+\t}\n+\n+\tif (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {\n+\t\tif (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||\n+\t\t\t\txform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {\n+\t\t\tif (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)\n+\t\t\t\treturn AESNI_MB_OP_AEAD_CIPHER_HASH;\n+\t\t\telse\n+\t\t\t\treturn AESNI_MB_OP_AEAD_HASH_CIPHER;\n+\t\t}\n+\t}\n+\n+\treturn AESNI_MB_OP_NOT_SUPPORTED;\n+}\n+\n+/** Set session authentication parameters */\n+static int\n+aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,\n+\t\tstruct aesni_mb_session *sess,\n+\t\tconst struct rte_crypto_sym_xform *xform)\n+{\n+\thash_one_block_t hash_oneblock_fn;\n+\tunsigned int key_larger_block_size = 0;\n+\tuint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };\n+\n+\tif (xform == NULL) {\n+\t\tsess->auth.algo = NULL_HASH;\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {\n+\t\tAESNI_MB_LOG(ERR, \"Crypto xform struct not of type auth\");\n+\t\treturn -1;\n+\t}\n+\n+\t/* Set the request digest size */\n+\tsess->auth.req_digest_len = xform->auth.digest_length;\n+\n+\t/* Select auth generate/verify */\n+\tsess->auth.operation = xform->auth.op;\n+\n+\t/* Set Authentication Parameters */\n+\tif (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {\n+\t\tsess->auth.algo = AES_XCBC;\n+\n+\t\tuint16_t xcbc_mac_digest_len =\n+\t\t\tget_truncated_digest_byte_length(AES_XCBC);\n+\t\tif (sess->auth.req_digest_len != xcbc_mac_digest_len) {\n+\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t\tIMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,\n+\t\t\t\tsess->auth.xcbc.k1_expanded,\n+\t\t\t\tsess->auth.xcbc.k2, sess->auth.xcbc.k3);\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {\n+\t\tuint32_t dust[4*15];\n+\n+\t\tsess->auth.algo = AES_CMAC;\n+\n+\t\tuint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);\n+\n+\t\tif (sess->auth.req_digest_len > cmac_digest_len) {\n+\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\t/*\n+\t\t * Multi-buffer lib supports digest sizes from 4 to 16 bytes\n+\t\t * in version 0.50 and sizes of 12 and 16 bytes,\n+\t\t * in version 0.49.\n+\t\t * If size requested is different, generate the full digest\n+\t\t * (16 bytes) in a temporary location and then memcpy\n+\t\t * the requested number of bytes.\n+\t\t */\n+\t\tif (sess->auth.req_digest_len < 4)\n+\t\t\tsess->auth.gen_digest_len = cmac_digest_len;\n+\t\telse\n+\t\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,\n+\t\t\t\tsess->auth.cmac.expkey, dust);\n+\t\tIMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,\n+\t\t\t\tsess->auth.cmac.skey1, sess->auth.cmac.skey2);\n+\t\treturn 0;\n+\t}\n+\n+\tswitch (xform->auth.algo) {\n+\tcase RTE_CRYPTO_AUTH_MD5_HMAC:\n+\t\tsess->auth.algo = MD5;\n+\t\thash_oneblock_fn = mb_mgr->md5_one_block;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA1_HMAC:\n+\t\tsess->auth.algo = SHA1;\n+\t\thash_oneblock_fn = mb_mgr->sha1_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {\n+\t\t\tIMB_SHA1(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA224_HMAC:\n+\t\tsess->auth.algo = SHA_224;\n+\t\thash_oneblock_fn = mb_mgr->sha224_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {\n+\t\t\tIMB_SHA224(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA256_HMAC:\n+\t\tsess->auth.algo = SHA_256;\n+\t\thash_oneblock_fn = mb_mgr->sha256_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {\n+\t\t\tIMB_SHA256(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA384_HMAC:\n+\t\tsess->auth.algo = SHA_384;\n+\t\thash_oneblock_fn = mb_mgr->sha384_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {\n+\t\t\tIMB_SHA384(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA512_HMAC:\n+\t\tsess->auth.algo = SHA_512;\n+\t\thash_oneblock_fn = mb_mgr->sha512_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {\n+\t\t\tIMB_SHA512(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tdefault:\n+\t\tAESNI_MB_LOG(ERR, \"Unsupported authentication algorithm selection\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\tuint16_t trunc_digest_size =\n+\t\t\tget_truncated_digest_byte_length(sess->auth.algo);\n+\tuint16_t full_digest_size =\n+\t\t\tget_digest_byte_length(sess->auth.algo);\n+\n+\tif (sess->auth.req_digest_len > full_digest_size ||\n+\t\t\tsess->auth.req_digest_len == 0) {\n+\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (sess->auth.req_digest_len != trunc_digest_size &&\n+\t\t\tsess->auth.req_digest_len != full_digest_size)\n+\t\tsess->auth.gen_digest_len = full_digest_size;\n+\telse\n+\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t/* Calculate Authentication precomputes */\n+\tif (key_larger_block_size) {\n+\t\tcalculate_auth_precomputes(hash_oneblock_fn,\n+\t\t\tsess->auth.pads.inner, sess->auth.pads.outer,\n+\t\t\thashed_key,\n+\t\t\txform->auth.key.length,\n+\t\t\tget_auth_algo_blocksize(sess->auth.algo));\n+\t} else {\n+\t\tcalculate_auth_precomputes(hash_oneblock_fn,\n+\t\t\tsess->auth.pads.inner, sess->auth.pads.outer,\n+\t\t\txform->auth.key.data,\n+\t\t\txform->auth.key.length,\n+\t\t\tget_auth_algo_blocksize(sess->auth.algo));\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/** Set session cipher parameters */\n+static int\n+aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,\n+\t\tstruct aesni_mb_session *sess,\n+\t\tconst struct rte_crypto_sym_xform *xform)\n+{\n+\tuint8_t is_aes = 0;\n+\tuint8_t is_3DES = 0;\n+\n+\tif (xform == NULL) {\n+\t\tsess->cipher.mode = NULL_CIPHER;\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {\n+\t\tAESNI_MB_LOG(ERR, \"Crypto xform struct not of type cipher\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* Select cipher direction */\n+\tswitch (xform->cipher.op) {\n+\tcase RTE_CRYPTO_CIPHER_OP_ENCRYPT:\n+\t\tsess->cipher.direction = ENCRYPT;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_OP_DECRYPT:\n+\t\tsess->cipher.direction = DECRYPT;\n+\t\tbreak;\n+\tdefault:\n+\t\tAESNI_MB_LOG(ERR, \"Invalid cipher operation parameter\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* Select cipher mode */\n+\tswitch (xform->cipher.algo) {\n+\tcase RTE_CRYPTO_CIPHER_AES_CBC:\n+\t\tsess->cipher.mode = CBC;\n+\t\tis_aes = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_AES_CTR:\n+\t\tsess->cipher.mode = CNTR;\n+\t\tis_aes = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_AES_DOCSISBPI:\n+\t\tsess->cipher.mode = DOCSIS_SEC_BPI;\n+\t\tis_aes = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_DES_CBC:\n+\t\tsess->cipher.mode = DES;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_DES_DOCSISBPI:\n+\t\tsess->cipher.mode = DOCSIS_DES;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_3DES_CBC:\n+\t\tsess->cipher.mode = DES3;\n+\t\tis_3DES = 1;\n+\t\tbreak;\n+\tdefault:\n+\t\tAESNI_MB_LOG(ERR, \"Unsupported cipher mode parameter\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* Set IV parameters */\n+\tsess->iv.offset = xform->cipher.iv.offset;\n+\tsess->iv.length = xform->cipher.iv.length;\n+\n+\t/* Check key length and choose key expansion function for AES */\n+\tif (is_aes) {\n+\t\tswitch (xform->cipher.key.length) {\n+\t\tcase AES_128_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = AES_128_BYTES;\n+\t\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tcase AES_192_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = AES_192_BYTES;\n+\t\t\tIMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tcase AES_256_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = AES_256_BYTES;\n+\t\t\tIMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t} else if (is_3DES) {\n+\t\tuint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],\n+\t\t\t\tsess->cipher.exp_3des_keys.key[1],\n+\t\t\t\tsess->cipher.exp_3des_keys.key[2]};\n+\n+\t\tswitch (xform->cipher.key.length) {\n+\t\tcase  24:\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n+\t\t\t\t\txform->cipher.key.data);\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[1],\n+\t\t\t\t\txform->cipher.key.data + 8);\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[2],\n+\t\t\t\t\txform->cipher.key.data + 16);\n+\n+\t\t\t/* Initialize keys - 24 bytes: [K1-K2-K3] */\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];\n+\t\t\tbreak;\n+\t\tcase 16:\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n+\t\t\t\t\txform->cipher.key.data);\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[1],\n+\t\t\t\t\txform->cipher.key.data + 8);\n+\t\t\t/* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];\n+\t\t\tbreak;\n+\t\tcase 8:\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n+\t\t\t\t\txform->cipher.key.data);\n+\n+\t\t\t/* Initialize keys - 8 bytes: [K1 = K2 = K3] */\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\tsess->cipher.key_length_in_bytes = 24;\n+\t} else {\n+\t\tif (xform->cipher.key.length != 8) {\n+\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->cipher.key_length_in_bytes = 8;\n+\n+\t\tIMB_DES_KEYSCHED(mb_mgr,\n+\t\t\t(uint64_t *)sess->cipher.expanded_aes_keys.encode,\n+\t\t\t\txform->cipher.key.data);\n+\t\tIMB_DES_KEYSCHED(mb_mgr,\n+\t\t\t(uint64_t *)sess->cipher.expanded_aes_keys.decode,\n+\t\t\t\txform->cipher.key.data);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int\n+aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,\n+\t\tstruct aesni_mb_session *sess,\n+\t\tconst struct rte_crypto_sym_xform *xform)\n+{\n+\tswitch (xform->aead.op) {\n+\tcase RTE_CRYPTO_AEAD_OP_ENCRYPT:\n+\t\tsess->cipher.direction = ENCRYPT;\n+\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AEAD_OP_DECRYPT:\n+\t\tsess->cipher.direction = DECRYPT;\n+\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;\n+\t\tbreak;\n+\tdefault:\n+\t\tAESNI_MB_LOG(ERR, \"Invalid aead operation parameter\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tswitch (xform->aead.algo) {\n+\tcase RTE_CRYPTO_AEAD_AES_CCM:\n+\t\tsess->cipher.mode = CCM;\n+\t\tsess->auth.algo = AES_CCM;\n+\n+\t\t/* Check key length and choose key expansion function for AES */\n+\t\tswitch (xform->aead.key.length) {\n+\t\tcase AES_128_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = AES_128_BYTES;\n+\t\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\tbreak;\n+\n+\tcase RTE_CRYPTO_AEAD_AES_GCM:\n+\t\tsess->cipher.mode = GCM;\n+\t\tsess->auth.algo = AES_GMAC;\n+\n+\t\tswitch (xform->aead.key.length) {\n+\t\tcase AES_128_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = AES_128_BYTES;\n+\t\t\tIMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tbreak;\n+\t\tcase AES_192_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = AES_192_BYTES;\n+\t\t\tIMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tbreak;\n+\t\tcase AES_256_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = AES_256_BYTES;\n+\t\t\tIMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\tbreak;\n+\n+\tdefault:\n+\t\tAESNI_MB_LOG(ERR, \"Unsupported aead mode parameter\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* Set IV parameters */\n+\tsess->iv.offset = xform->aead.iv.offset;\n+\tsess->iv.length = xform->aead.iv.length;\n+\n+\tsess->auth.req_digest_len = xform->aead.digest_length;\n+\t/* CCM digests must be between 4 and 16 and an even number */\n+\tif (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||\n+\t\t\tsess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||\n+\t\t\t(sess->auth.req_digest_len & 1) == 1) {\n+\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\treturn 0;\n+}\n+\n+/** Parse crypto xform chain and set private session parameters */\n+int\n+aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,\n+\t\tstruct aesni_mb_session *sess,\n+\t\tconst struct rte_crypto_sym_xform *xform)\n+{\n+\tconst struct rte_crypto_sym_xform *auth_xform = NULL;\n+\tconst struct rte_crypto_sym_xform *cipher_xform = NULL;\n+\tconst struct rte_crypto_sym_xform *aead_xform = NULL;\n+\tint ret;\n+\n+\t/* Select Crypto operation - hash then cipher / cipher then hash */\n+\tswitch (aesni_mb_get_chain_order(xform)) {\n+\tcase AESNI_MB_OP_HASH_CIPHER:\n+\t\tsess->chain_order = HASH_CIPHER;\n+\t\tauth_xform = xform;\n+\t\tcipher_xform = xform->next;\n+\t\tbreak;\n+\tcase AESNI_MB_OP_CIPHER_HASH:\n+\t\tsess->chain_order = CIPHER_HASH;\n+\t\tauth_xform = xform->next;\n+\t\tcipher_xform = xform;\n+\t\tbreak;\n+\tcase AESNI_MB_OP_HASH_ONLY:\n+\t\tsess->chain_order = HASH_CIPHER;\n+\t\tauth_xform = xform;\n+\t\tcipher_xform = NULL;\n+\t\tbreak;\n+\tcase AESNI_MB_OP_CIPHER_ONLY:\n+\t\t/*\n+\t\t * Multi buffer library operates only at two modes,\n+\t\t * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,\n+\t\t * chain order depends on cipher operation: encryption is always\n+\t\t * the first operation and decryption the last one.\n+\t\t */\n+\t\tif (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)\n+\t\t\tsess->chain_order = CIPHER_HASH;\n+\t\telse\n+\t\t\tsess->chain_order = HASH_CIPHER;\n+\t\tauth_xform = NULL;\n+\t\tcipher_xform = xform;\n+\t\tbreak;\n+\tcase AESNI_MB_OP_AEAD_CIPHER_HASH:\n+\t\tsess->chain_order = CIPHER_HASH;\n+\t\tsess->aead.aad_len = xform->aead.aad_length;\n+\t\taead_xform = xform;\n+\t\tbreak;\n+\tcase AESNI_MB_OP_AEAD_HASH_CIPHER:\n+\t\tsess->chain_order = HASH_CIPHER;\n+\t\tsess->aead.aad_len = xform->aead.aad_length;\n+\t\taead_xform = xform;\n+\t\tbreak;\n+\tcase AESNI_MB_OP_NOT_SUPPORTED:\n+\tdefault:\n+\t\tAESNI_MB_LOG(ERR, \"Unsupported operation chain order parameter\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* Default IV length = 0 */\n+\tsess->iv.length = 0;\n+\n+\tret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);\n+\tif (ret != 0) {\n+\t\tAESNI_MB_LOG(ERR, \"Invalid/unsupported authentication parameters\");\n+\t\treturn ret;\n+\t}\n+\n+\tret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,\n+\t\t\tcipher_xform);\n+\tif (ret != 0) {\n+\t\tAESNI_MB_LOG(ERR, \"Invalid/unsupported cipher parameters\");\n+\t\treturn ret;\n+\t}\n+\n+\tif (aead_xform) {\n+\t\tret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,\n+\t\t\t\taead_xform);\n+\t\tif (ret != 0) {\n+\t\t\tAESNI_MB_LOG(ERR, \"Invalid/unsupported aead parameters\");\n+\t\t\treturn ret;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * burst enqueue, place crypto operations on ingress queue for processing.\n+ *\n+ * @param __qp         Queue Pair to process\n+ * @param ops          Crypto operations for processing\n+ * @param nb_ops       Number of crypto operations for processing\n+ *\n+ * @return\n+ * - Number of crypto operations enqueued\n+ */\n+static uint16_t\n+aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,\n+\t\tuint16_t nb_ops)\n+{\n+\tstruct aesni_mb_qp *qp = __qp;\n+\n+\tunsigned int nb_enqueued;\n+\n+\tnb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,\n+\t\t\t(void **)ops, nb_ops, NULL);\n+\n+\tqp->stats.enqueued_count += nb_enqueued;\n+\n+\treturn nb_enqueued;\n+}\n+\n+/** Get multi buffer session */\n+static inline struct aesni_mb_session *\n+get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)\n+{\n+\tstruct aesni_mb_session *sess = NULL;\n+\n+\tif (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {\n+\t\tif (likely(op->sym->session != NULL))\n+\t\t\tsess = (struct aesni_mb_session *)\n+\t\t\t\t\tget_sym_session_private_data(\n+\t\t\t\t\top->sym->session,\n+\t\t\t\t\tcryptodev_driver_id);\n+\t} else {\n+\t\tvoid *_sess = NULL;\n+\t\tvoid *_sess_private_data = NULL;\n+\n+\t\tif (rte_mempool_get(qp->sess_mp, (void **)&_sess))\n+\t\t\treturn NULL;\n+\n+\t\tif (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))\n+\t\t\treturn NULL;\n+\n+\t\tsess = (struct aesni_mb_session *)_sess_private_data;\n+\n+\t\tif (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,\n+\t\t\t\tsess, op->sym->xform) != 0)) {\n+\t\t\trte_mempool_put(qp->sess_mp, _sess);\n+\t\t\trte_mempool_put(qp->sess_mp, _sess_private_data);\n+\t\t\tsess = NULL;\n+\t\t}\n+\t\top->sym->session = (struct rte_cryptodev_sym_session *)_sess;\n+\t\tset_sym_session_private_data(op->sym->session,\n+\t\t\t\tcryptodev_driver_id, _sess_private_data);\n+\t}\n+\n+\tif (unlikely(sess == NULL))\n+\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\n+\treturn sess;\n+}\n+\n+/**\n+ * Process a crypto operation and complete a JOB_AES_HMAC job structure for\n+ * submission to the multi buffer library for processing.\n+ *\n+ * @param\tqp\tqueue pair\n+ * @param\tjob\tJOB_AES_HMAC structure to fill\n+ * @param\tm\tmbuf to process\n+ *\n+ * @return\n+ * - Completed JOB_AES_HMAC structure pointer on success\n+ * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible\n+ */\n+static inline int\n+set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,\n+\t\tstruct rte_crypto_op *op, uint8_t *digest_idx)\n+{\n+\tstruct rte_mbuf *m_src = op->sym->m_src, *m_dst;\n+\tstruct aesni_mb_session *session;\n+\tuint16_t m_offset = 0;\n+\n+\tsession = get_session(qp, op);\n+\tif (session == NULL) {\n+\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\t\treturn -1;\n+\t}\n+\n+\t/* Set crypto operation */\n+\tjob->chain_order = session->chain_order;\n+\n+\t/* Set cipher parameters */\n+\tjob->cipher_direction = session->cipher.direction;\n+\tjob->cipher_mode = session->cipher.mode;\n+\n+\tjob->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;\n+\n+\t/* Set authentication parameters */\n+\tjob->hash_alg = session->auth.algo;\n+\n+\tswitch (job->hash_alg) {\n+\tcase AES_XCBC:\n+\t\tjob->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;\n+\t\tjob->u.XCBC._k2 = session->auth.xcbc.k2;\n+\t\tjob->u.XCBC._k3 = session->auth.xcbc.k3;\n+\n+\t\tjob->aes_enc_key_expanded =\n+\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n+\t\tjob->aes_dec_key_expanded =\n+\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n+\t\tbreak;\n+\n+\tcase AES_CCM:\n+\t\tjob->u.CCM.aad = op->sym->aead.aad.data + 18;\n+\t\tjob->u.CCM.aad_len_in_bytes = session->aead.aad_len;\n+\t\tjob->aes_enc_key_expanded =\n+\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n+\t\tjob->aes_dec_key_expanded =\n+\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n+\t\tbreak;\n+\n+\tcase AES_CMAC:\n+\t\tjob->u.CMAC._key_expanded = session->auth.cmac.expkey;\n+\t\tjob->u.CMAC._skey1 = session->auth.cmac.skey1;\n+\t\tjob->u.CMAC._skey2 = session->auth.cmac.skey2;\n+\t\tjob->aes_enc_key_expanded =\n+\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n+\t\tjob->aes_dec_key_expanded =\n+\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n+\t\tbreak;\n+\n+\tcase AES_GMAC:\n+\t\tjob->u.GCM.aad = op->sym->aead.aad.data;\n+\t\tjob->u.GCM.aad_len_in_bytes = session->aead.aad_len;\n+\t\tjob->aes_enc_key_expanded = &session->cipher.gcm_key;\n+\t\tjob->aes_dec_key_expanded = &session->cipher.gcm_key;\n+\t\tbreak;\n+\n+\tdefault:\n+\t\tjob->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;\n+\t\tjob->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;\n+\n+\t\tif (job->cipher_mode == DES3) {\n+\t\t\tjob->aes_enc_key_expanded =\n+\t\t\t\tsession->cipher.exp_3des_keys.ks_ptr;\n+\t\t\tjob->aes_dec_key_expanded =\n+\t\t\t\tsession->cipher.exp_3des_keys.ks_ptr;\n+\t\t} else {\n+\t\t\tjob->aes_enc_key_expanded =\n+\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n+\t\t\tjob->aes_dec_key_expanded =\n+\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n+\t\t}\n+\t}\n+\n+\t/* Mutable crypto operation parameters */\n+\tif (op->sym->m_dst) {\n+\t\tm_src = m_dst = op->sym->m_dst;\n+\n+\t\t/* append space for output data to mbuf */\n+\t\tchar *odata = rte_pktmbuf_append(m_dst,\n+\t\t\t\trte_pktmbuf_data_len(op->sym->m_src));\n+\t\tif (odata == NULL) {\n+\t\t\tAESNI_MB_LOG(ERR, \"failed to allocate space in destination \"\n+\t\t\t\t\t\"mbuf for source data\");\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t\t\treturn -1;\n+\t\t}\n+\n+\t\tmemcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*),\n+\t\t\t\trte_pktmbuf_data_len(op->sym->m_src));\n+\t} else {\n+\t\tm_dst = m_src;\n+\t\tif (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC)\n+\t\t\tm_offset = op->sym->aead.data.offset;\n+\t\telse\n+\t\t\tm_offset = op->sym->cipher.data.offset;\n+\t}\n+\n+\t/* Set digest output location */\n+\tif (job->hash_alg != NULL_HASH &&\n+\t\t\tsession->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {\n+\t\tjob->auth_tag_output = qp->temp_digests[*digest_idx];\n+\t\t*digest_idx = (*digest_idx + 1) % MAX_JOBS;\n+\t} else {\n+\t\tif (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC)\n+\t\t\tjob->auth_tag_output = op->sym->aead.digest.data;\n+\t\telse\n+\t\t\tjob->auth_tag_output = op->sym->auth.digest.data;\n+\n+\t\tif (session->auth.req_digest_len != session->auth.gen_digest_len) {\n+\t\t\tjob->auth_tag_output = qp->temp_digests[*digest_idx];\n+\t\t\t*digest_idx = (*digest_idx + 1) % MAX_JOBS;\n+\t\t}\n+\t}\n+\t/*\n+\t * Multi-buffer library current only support returning a truncated\n+\t * digest length as specified in the relevant IPsec RFCs\n+\t */\n+\n+\t/* Set digest length */\n+\tjob->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;\n+\n+\t/* Set IV parameters */\n+\tjob->iv_len_in_bytes = session->iv.length;\n+\n+\t/* Data  Parameter */\n+\tjob->src = rte_pktmbuf_mtod(m_src, uint8_t *);\n+\tjob->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);\n+\n+\tswitch (job->hash_alg) {\n+\tcase AES_CCM:\n+\t\tjob->cipher_start_src_offset_in_bytes =\n+\t\t\t\top->sym->aead.data.offset;\n+\t\tjob->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;\n+\t\tjob->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;\n+\t\tjob->msg_len_to_hash_in_bytes = op->sym->aead.data.length;\n+\n+\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\tsession->iv.offset + 1);\n+\t\tbreak;\n+\n+\tcase AES_GMAC:\n+\t\tjob->cipher_start_src_offset_in_bytes =\n+\t\t\t\top->sym->aead.data.offset;\n+\t\tjob->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;\n+\t\tjob->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;\n+\t\tjob->msg_len_to_hash_in_bytes = job->msg_len_to_cipher_in_bytes;\n+\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\t\tsession->iv.offset);\n+\t\tbreak;\n+\n+\tdefault:\n+\t\tjob->cipher_start_src_offset_in_bytes =\n+\t\t\t\top->sym->cipher.data.offset;\n+\t\tjob->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;\n+\n+\t\tjob->hash_start_src_offset_in_bytes = op->sym->auth.data.offset;\n+\t\tjob->msg_len_to_hash_in_bytes = op->sym->auth.data.length;\n+\n+\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\tsession->iv.offset);\n+\t}\n+\n+\t/* Set user data to be crypto operation data struct */\n+\tjob->user_data = op;\n+\n+\treturn 0;\n+}\n+\n+static inline void\n+verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,\n+\t\tstruct aesni_mb_session *sess)\n+{\n+\t/* Verify digest if required */\n+\tif (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) {\n+\t\tif (memcmp(job->auth_tag_output, op->sym->aead.digest.data,\n+\t\t\t\tsess->auth.req_digest_len) != 0)\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n+\t} else {\n+\t\tif (memcmp(job->auth_tag_output, op->sym->auth.digest.data,\n+\t\t\t\tsess->auth.req_digest_len) != 0)\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n+\t}\n+}\n+\n+static inline void\n+generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,\n+\t\tstruct aesni_mb_session *sess)\n+{\n+\t/* No extra copy neeed */\n+\tif (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))\n+\t\treturn;\n+\n+\t/*\n+\t * This can only happen for HMAC, so only digest\n+\t * for authentication algos is required\n+\t */\n+\tmemcpy(op->sym->auth.digest.data, job->auth_tag_output,\n+\t\t\tsess->auth.req_digest_len);\n+}\n+\n+/**\n+ * Process a completed job and return rte_mbuf which job processed\n+ *\n+ * @param qp\t\tQueue Pair to process\n+ * @param job\tJOB_AES_HMAC job to process\n+ *\n+ * @return\n+ * - Returns processed crypto operation.\n+ * - Returns NULL on invalid job\n+ */\n+static inline struct rte_crypto_op *\n+post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)\n+{\n+\tstruct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;\n+\tstruct aesni_mb_session *sess = get_sym_session_private_data(\n+\t\t\t\t\t\t\top->sym->session,\n+\t\t\t\t\t\t\tcryptodev_driver_id);\n+\n+\tif (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {\n+\t\tswitch (job->status) {\n+\t\tcase STS_COMPLETED:\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_SUCCESS;\n+\n+\t\t\tif (job->hash_alg != NULL_HASH) {\n+\t\t\t\tif (sess->auth.operation ==\n+\t\t\t\t\t\tRTE_CRYPTO_AUTH_OP_VERIFY)\n+\t\t\t\t\tverify_digest(job, op, sess);\n+\t\t\t\telse\n+\t\t\t\t\tgenerate_digest(job, op, sess);\n+\t\t\t}\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t\t}\n+\t}\n+\n+\t/* Free session if a session-less crypto op */\n+\tif (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {\n+\t\tmemset(sess, 0, sizeof(struct aesni_mb_session));\n+\t\tmemset(op->sym->session, 0,\n+\t\t\t\trte_cryptodev_sym_get_header_session_size());\n+\t\trte_mempool_put(qp->sess_mp, sess);\n+\t\trte_mempool_put(qp->sess_mp, op->sym->session);\n+\t\top->sym->session = NULL;\n+\t}\n+\n+\treturn op;\n+}\n+\n+/**\n+ * Process a completed JOB_AES_HMAC job and keep processing jobs until\n+ * get_completed_job return NULL\n+ *\n+ * @param qp\t\tQueue Pair to process\n+ * @param job\t\tJOB_AES_HMAC job\n+ *\n+ * @return\n+ * - Number of processed jobs\n+ */\n+static unsigned\n+handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,\n+\t\tstruct rte_crypto_op **ops, uint16_t nb_ops)\n+{\n+\tstruct rte_crypto_op *op = NULL;\n+\tunsigned processed_jobs = 0;\n+\n+\twhile (job != NULL) {\n+\t\top = post_process_mb_job(qp, job);\n+\n+\t\tif (op) {\n+\t\t\tops[processed_jobs++] = op;\n+\t\t\tqp->stats.dequeued_count++;\n+\t\t} else {\n+\t\t\tqp->stats.dequeue_err_count++;\n+\t\t\tbreak;\n+\t\t}\n+\t\tif (processed_jobs == nb_ops)\n+\t\t\tbreak;\n+\n+\t\tjob = IMB_GET_COMPLETED_JOB(qp->mb_mgr);\n+\t}\n+\n+\treturn processed_jobs;\n+}\n+\n+static inline uint16_t\n+flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,\n+\t\tuint16_t nb_ops)\n+{\n+\tint processed_ops = 0;\n+\n+\t/* Flush the remaining jobs */\n+\tJOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);\n+\n+\tif (job)\n+\t\tprocessed_ops += handle_completed_jobs(qp, job,\n+\t\t\t\t&ops[processed_ops], nb_ops - processed_ops);\n+\n+\treturn processed_ops;\n+}\n+\n+static inline JOB_AES_HMAC *\n+set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)\n+{\n+\tjob->chain_order = HASH_CIPHER;\n+\tjob->cipher_mode = NULL_CIPHER;\n+\tjob->hash_alg = NULL_HASH;\n+\tjob->cipher_direction = DECRYPT;\n+\n+\t/* Set user data to be crypto operation data struct */\n+\tjob->user_data = op;\n+\n+\treturn job;\n+}\n+\n+static uint16_t\n+aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,\n+\t\tuint16_t nb_ops)\n+{\n+\tstruct aesni_mb_qp *qp = queue_pair;\n+\n+\tstruct rte_crypto_op *op;\n+\tJOB_AES_HMAC *job;\n+\n+\tint retval, processed_jobs = 0;\n+\n+\tif (unlikely(nb_ops == 0))\n+\t\treturn 0;\n+\n+\tuint8_t digest_idx = qp->digest_idx;\n+\tdo {\n+\t\t/* Get next free mb job struct from mb manager */\n+\t\tjob = IMB_GET_NEXT_JOB(qp->mb_mgr);\n+\t\tif (unlikely(job == NULL)) {\n+\t\t\t/* if no free mb job structs we need to flush mb_mgr */\n+\t\t\tprocessed_jobs += flush_mb_mgr(qp,\n+\t\t\t\t\t&ops[processed_jobs],\n+\t\t\t\t\tnb_ops - processed_jobs);\n+\n+\t\t\tif (nb_ops == processed_jobs)\n+\t\t\t\tbreak;\n+\n+\t\t\tjob = IMB_GET_NEXT_JOB(qp->mb_mgr);\n+\t\t}\n+\n+\t\t/*\n+\t\t * Get next operation to process from ingress queue.\n+\t\t * There is no need to return the job to the MB_MGR\n+\t\t * if there are no more operations to process, since the MB_MGR\n+\t\t * can use that pointer again in next get_next calls.\n+\t\t */\n+\t\tretval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);\n+\t\tif (retval < 0)\n+\t\t\tbreak;\n+\n+\t\tretval = set_mb_job_params(job, qp, op, &digest_idx);\n+\t\tif (unlikely(retval != 0)) {\n+\t\t\tqp->stats.dequeue_err_count++;\n+\t\t\tset_job_null_op(job, op);\n+\t\t}\n+\n+\t\t/* Submit job to multi-buffer for processing */\n+#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG\n+\t\tjob = IMB_SUBMIT_JOB(qp->mb_mgr);\n+#else\n+\t\tjob = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);\n+#endif\n+\t\t/*\n+\t\t * If submit returns a processed job then handle it,\n+\t\t * before submitting subsequent jobs\n+\t\t */\n+\t\tif (job)\n+\t\t\tprocessed_jobs += handle_completed_jobs(qp, job,\n+\t\t\t\t\t&ops[processed_jobs],\n+\t\t\t\t\tnb_ops - processed_jobs);\n+\n+\t} while (processed_jobs < nb_ops);\n+\n+\tqp->digest_idx = digest_idx;\n+\n+\tif (processed_jobs < 1)\n+\t\tprocessed_jobs += flush_mb_mgr(qp,\n+\t\t\t\t&ops[processed_jobs],\n+\t\t\t\tnb_ops - processed_jobs);\n+\n+\treturn processed_jobs;\n+}\n+\n+static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);\n+\n+static int\n+cryptodev_aesni_mb_create(const char *name,\n+\t\t\tstruct rte_vdev_device *vdev,\n+\t\t\tstruct rte_cryptodev_pmd_init_params *init_params)\n+{\n+\tstruct rte_cryptodev *dev;\n+\tstruct aesni_mb_private *internals;\n+\tenum aesni_mb_vector_mode vector_mode;\n+\tMB_MGR *mb_mgr;\n+\n+\t/* Check CPU for support for AES instruction set */\n+\tif (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {\n+\t\tAESNI_MB_LOG(ERR, \"AES instructions not supported by CPU\");\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);\n+\tif (dev == NULL) {\n+\t\tAESNI_MB_LOG(ERR, \"failed to create cryptodev vdev\");\n+\t\treturn -ENODEV;\n+\t}\n+\n+\t/* Check CPU for supported vector instruction set */\n+\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))\n+\t\tvector_mode = RTE_AESNI_MB_AVX512;\n+\telse if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))\n+\t\tvector_mode = RTE_AESNI_MB_AVX2;\n+\telse if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))\n+\t\tvector_mode = RTE_AESNI_MB_AVX;\n+\telse\n+\t\tvector_mode = RTE_AESNI_MB_SSE;\n+\n+\tdev->driver_id = cryptodev_driver_id;\n+\tdev->dev_ops = rte_aesni_mb_pmd_ops;\n+\n+\t/* register rx/tx burst functions for data path */\n+\tdev->dequeue_burst = aesni_mb_pmd_dequeue_burst;\n+\tdev->enqueue_burst = aesni_mb_pmd_enqueue_burst;\n+\n+\tdev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |\n+\t\t\tRTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |\n+\t\t\tRTE_CRYPTODEV_FF_CPU_AESNI;\n+\n+\tmb_mgr = alloc_mb_mgr(0);\n+\tif (mb_mgr == NULL)\n+\t\treturn -ENOMEM;\n+\n+\tswitch (vector_mode) {\n+\tcase RTE_AESNI_MB_SSE:\n+\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;\n+\t\tinit_mb_mgr_sse(mb_mgr);\n+\t\tbreak;\n+\tcase RTE_AESNI_MB_AVX:\n+\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;\n+\t\tinit_mb_mgr_avx(mb_mgr);\n+\t\tbreak;\n+\tcase RTE_AESNI_MB_AVX2:\n+\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;\n+\t\tinit_mb_mgr_avx2(mb_mgr);\n+\t\tbreak;\n+\tcase RTE_AESNI_MB_AVX512:\n+\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;\n+\t\tinit_mb_mgr_avx512(mb_mgr);\n+\t\tbreak;\n+\tdefault:\n+\t\tAESNI_MB_LOG(ERR, \"Unsupported vector mode %u\\n\", vector_mode);\n+\t\tgoto error_exit;\n+\t}\n+\n+\t/* Set vector instructions mode supported */\n+\tinternals = dev->data->dev_private;\n+\n+\tinternals->vector_mode = vector_mode;\n+\tinternals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;\n+\tinternals->mb_mgr = mb_mgr;\n+\n+\tAESNI_MB_LOG(INFO, \"IPSec Multi-buffer library version used: %s\\n\",\n+\t\t\timb_get_version_str());\n+\n+\treturn 0;\n+\n+error_exit:\n+\tif (mb_mgr)\n+\t\tfree_mb_mgr(mb_mgr);\n+\n+\trte_cryptodev_pmd_destroy(dev);\n+\n+\treturn -1;\n+}\n+\n+static int\n+cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)\n+{\n+\tstruct rte_cryptodev_pmd_init_params init_params = {\n+\t\t\"\",\n+\t\tsizeof(struct aesni_mb_private),\n+\t\trte_socket_id(),\n+\t\tRTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS\n+\t};\n+\tconst char *name, *args;\n+\tint retval;\n+\n+\tname = rte_vdev_device_name(vdev);\n+\tif (name == NULL)\n+\t\treturn -EINVAL;\n+\n+\targs = rte_vdev_device_args(vdev);\n+\n+\tretval = rte_cryptodev_pmd_parse_input_args(&init_params, args);\n+\tif (retval) {\n+\t\tAESNI_MB_LOG(ERR, \"Failed to parse initialisation arguments[%s]\",\n+\t\t\t\targs);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\treturn cryptodev_aesni_mb_create(name, vdev, &init_params);\n+}\n+\n+static int\n+cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)\n+{\n+\tstruct rte_cryptodev *cryptodev;\n+\tstruct aesni_mb_private *internals;\n+\tconst char *name;\n+\n+\tname = rte_vdev_device_name(vdev);\n+\tif (name == NULL)\n+\t\treturn -EINVAL;\n+\n+\tcryptodev = rte_cryptodev_pmd_get_named_dev(name);\n+\tif (cryptodev == NULL)\n+\t\treturn -ENODEV;\n+\n+\tinternals = cryptodev->data->dev_private;\n+\n+\tfree_mb_mgr(internals->mb_mgr);\n+\n+\treturn rte_cryptodev_pmd_destroy(cryptodev);\n+}\n+\n+static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {\n+\t.probe = cryptodev_aesni_mb_probe,\n+\t.remove = cryptodev_aesni_mb_remove\n+};\n+\n+static struct cryptodev_driver aesni_mb_crypto_drv;\n+\n+RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);\n+RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);\n+RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,\n+\t\"max_nb_queue_pairs=<int> \"\n+\t\"socket_id=<int>\");\n+RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,\n+\t\tcryptodev_aesni_mb_pmd_drv.driver,\n+\t\tcryptodev_driver_id);\n+\n+RTE_INIT(aesni_mb_init_log)\n+{\n+\taesni_mb_logtype_driver = rte_log_register(\"pmd.crypto.aesni_mb\");\n+}\ndiff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c\nnew file mode 100644\nindex 000000000..5788e37d1\n--- /dev/null\n+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c\n@@ -0,0 +1,681 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(c) 2015-2017 Intel Corporation\n+ */\n+\n+#include <string.h>\n+\n+#include <rte_common.h>\n+#include <rte_malloc.h>\n+#include <rte_cryptodev_pmd.h>\n+\n+#include \"rte_aesni_mb_pmd_private.h\"\n+\n+\n+static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {\n+\t{\t/* MD5 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_MD5_HMAC,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 64,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA1 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA1_HMAC,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 20,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA224 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 28,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA256 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA256_HMAC,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA384 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA384_HMAC,\n+\t\t\t\t.block_size = 128,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 48,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA512 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA512_HMAC,\n+\t\t\t\t.block_size = 128,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 64,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES XCBC HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 12,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES CBC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_CBC,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES CTR */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_CTR,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 4\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES DOCSIS BPI */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* DES CBC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_DES_CBC,\n+\t\t\t\t.block_size = 8,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/*  3DES CBC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_3DES_CBC,\n+\t\t\t\t.block_size = 8,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 24,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* DES DOCSIS BPI */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,\n+\t\t\t\t.block_size = 8,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES CCM */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n+\t\t\t{.aead = {\n+\t\t\t\t.algo = RTE_CRYPTO_AEAD_AES_CCM,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 4,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 2\n+\t\t\t\t},\n+\t\t\t\t.aad_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 46,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 7,\n+\t\t\t\t\t.max = 13,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES CMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_AES_CMAC,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES GCM */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n+\t\t\t{.aead = {\n+\t\t\t\t.algo = RTE_CRYPTO_AEAD_AES_GCM,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 4\n+\t\t\t\t},\n+\t\t\t\t.aad_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 12,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\tRTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()\n+};\n+\n+\n+/** Configure device */\n+static int\n+aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,\n+\t\t__rte_unused struct rte_cryptodev_config *config)\n+{\n+\treturn 0;\n+}\n+\n+/** Start device */\n+static int\n+aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)\n+{\n+\treturn 0;\n+}\n+\n+/** Stop device */\n+static void\n+aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)\n+{\n+}\n+\n+/** Close device */\n+static int\n+aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)\n+{\n+\treturn 0;\n+}\n+\n+\n+/** Get device statistics */\n+static void\n+aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,\n+\t\tstruct rte_cryptodev_stats *stats)\n+{\n+\tint qp_id;\n+\n+\tfor (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {\n+\t\tstruct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];\n+\n+\t\tstats->enqueued_count += qp->stats.enqueued_count;\n+\t\tstats->dequeued_count += qp->stats.dequeued_count;\n+\n+\t\tstats->enqueue_err_count += qp->stats.enqueue_err_count;\n+\t\tstats->dequeue_err_count += qp->stats.dequeue_err_count;\n+\t}\n+}\n+\n+/** Reset device statistics */\n+static void\n+aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)\n+{\n+\tint qp_id;\n+\n+\tfor (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {\n+\t\tstruct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];\n+\n+\t\tmemset(&qp->stats, 0, sizeof(qp->stats));\n+\t}\n+}\n+\n+\n+/** Get device info */\n+static void\n+aesni_mb_pmd_info_get(struct rte_cryptodev *dev,\n+\t\tstruct rte_cryptodev_info *dev_info)\n+{\n+\tstruct aesni_mb_private *internals = dev->data->dev_private;\n+\n+\tif (dev_info != NULL) {\n+\t\tdev_info->driver_id = dev->driver_id;\n+\t\tdev_info->feature_flags = dev->feature_flags;\n+\t\tdev_info->capabilities = aesni_mb_pmd_capabilities;\n+\t\tdev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;\n+\t\t/* No limit of number of sessions */\n+\t\tdev_info->sym.max_nb_sessions = 0;\n+\t}\n+}\n+\n+/** Release queue pair */\n+static int\n+aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)\n+{\n+\tstruct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];\n+\tstruct rte_ring *r = NULL;\n+\n+\tif (qp != NULL) {\n+\t\tr = rte_ring_lookup(qp->name);\n+\t\tif (r)\n+\t\t\trte_ring_free(r);\n+\t\tif (qp->mb_mgr)\n+\t\t\tfree_mb_mgr(qp->mb_mgr);\n+\t\trte_free(qp);\n+\t\tdev->data->queue_pairs[qp_id] = NULL;\n+\t}\n+\treturn 0;\n+}\n+\n+/** set a unique name for the queue pair based on it's name, dev_id and qp_id */\n+static int\n+aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,\n+\t\tstruct aesni_mb_qp *qp)\n+{\n+\tunsigned n = snprintf(qp->name, sizeof(qp->name),\n+\t\t\t\"aesni_mb_pmd_%u_qp_%u\",\n+\t\t\tdev->data->dev_id, qp->id);\n+\n+\tif (n >= sizeof(qp->name))\n+\t\treturn -1;\n+\n+\treturn 0;\n+}\n+\n+/** Create a ring to place processed operations on */\n+static struct rte_ring *\n+aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,\n+\t\tunsigned int ring_size, int socket_id)\n+{\n+\tstruct rte_ring *r;\n+\tchar ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];\n+\n+\tunsigned int n = snprintf(ring_name, sizeof(ring_name), \"%s\", qp->name);\n+\n+\tif (n >= sizeof(ring_name))\n+\t\treturn NULL;\n+\n+\tr = rte_ring_lookup(ring_name);\n+\tif (r) {\n+\t\tif (rte_ring_get_size(r) >= ring_size) {\n+\t\t\tAESNI_MB_LOG(INFO, \"Reusing existing ring %s for processed ops\",\n+\t\t\tring_name);\n+\t\t\treturn r;\n+\t\t}\n+\n+\t\tAESNI_MB_LOG(ERR, \"Unable to reuse existing ring %s for processed ops\",\n+\t\t\tring_name);\n+\t\treturn NULL;\n+\t}\n+\n+\treturn rte_ring_create(ring_name, ring_size, socket_id,\n+\t\t\tRING_F_SP_ENQ | RING_F_SC_DEQ);\n+}\n+\n+/** Setup a queue pair */\n+static int\n+aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,\n+\t\tconst struct rte_cryptodev_qp_conf *qp_conf,\n+\t\tint socket_id, struct rte_mempool *session_pool)\n+{\n+\tstruct aesni_mb_qp *qp = NULL;\n+\tstruct aesni_mb_private *internals = dev->data->dev_private;\n+\tint ret = -1;\n+\n+\t/* Free memory prior to re-allocation if needed. */\n+\tif (dev->data->queue_pairs[qp_id] != NULL)\n+\t\taesni_mb_pmd_qp_release(dev, qp_id);\n+\n+\t/* Allocate the queue pair data structure. */\n+\tqp = rte_zmalloc_socket(\"AES-NI PMD Queue Pair\", sizeof(*qp),\n+\t\t\t\t\tRTE_CACHE_LINE_SIZE, socket_id);\n+\tif (qp == NULL)\n+\t\treturn -ENOMEM;\n+\n+\tqp->id = qp_id;\n+\tdev->data->queue_pairs[qp_id] = qp;\n+\n+\tif (aesni_mb_pmd_qp_set_unique_name(dev, qp))\n+\t\tgoto qp_setup_cleanup;\n+\n+\n+\tqp->mb_mgr = alloc_mb_mgr(0);\n+\tif (qp->mb_mgr == NULL) {\n+\t\tret = -ENOMEM;\n+\t\tgoto qp_setup_cleanup;\n+\t}\n+\n+\tswitch (internals->vector_mode) {\n+\tcase RTE_AESNI_MB_SSE:\n+\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;\n+\t\tinit_mb_mgr_sse(qp->mb_mgr);\n+\t\tbreak;\n+\tcase RTE_AESNI_MB_AVX:\n+\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;\n+\t\tinit_mb_mgr_avx(qp->mb_mgr);\n+\t\tbreak;\n+\tcase RTE_AESNI_MB_AVX2:\n+\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;\n+\t\tinit_mb_mgr_avx2(qp->mb_mgr);\n+\t\tbreak;\n+\tcase RTE_AESNI_MB_AVX512:\n+\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;\n+\t\tinit_mb_mgr_avx512(qp->mb_mgr);\n+\t\tbreak;\n+\tdefault:\n+\t\tAESNI_MB_LOG(ERR, \"Unsupported vector mode %u\\n\",\n+\t\t\t\tinternals->vector_mode);\n+\t\tgoto qp_setup_cleanup;\n+\t}\n+\n+\tqp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,\n+\t\t\tqp_conf->nb_descriptors, socket_id);\n+\tif (qp->ingress_queue == NULL) {\n+\t\tret = -1;\n+\t\tgoto qp_setup_cleanup;\n+\t}\n+\n+\tqp->sess_mp = session_pool;\n+\n+\tmemset(&qp->stats, 0, sizeof(qp->stats));\n+\n+\tchar mp_name[RTE_MEMPOOL_NAMESIZE];\n+\n+\tsnprintf(mp_name, RTE_MEMPOOL_NAMESIZE,\n+\t\t\t\t\"digest_mp_%u_%u\", dev->data->dev_id, qp_id);\n+\treturn 0;\n+\n+qp_setup_cleanup:\n+\tif (qp) {\n+\t\tif (qp->mb_mgr)\n+\t\t\tfree_mb_mgr(qp->mb_mgr);\n+\t\trte_free(qp);\n+\t}\n+\n+\treturn ret;\n+}\n+\n+/** Return the number of allocated queue pairs */\n+static uint32_t\n+aesni_mb_pmd_qp_count(struct rte_cryptodev *dev)\n+{\n+\treturn dev->data->nb_queue_pairs;\n+}\n+\n+/** Returns the size of the aesni multi-buffer session structure */\n+static unsigned\n+aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)\n+{\n+\treturn sizeof(struct aesni_mb_session);\n+}\n+\n+/** Configure a aesni multi-buffer session from a crypto xform chain */\n+static int\n+aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,\n+\t\tstruct rte_crypto_sym_xform *xform,\n+\t\tstruct rte_cryptodev_sym_session *sess,\n+\t\tstruct rte_mempool *mempool)\n+{\n+\tvoid *sess_private_data;\n+\tstruct aesni_mb_private *internals = dev->data->dev_private;\n+\tint ret;\n+\n+\tif (unlikely(sess == NULL)) {\n+\t\tAESNI_MB_LOG(ERR, \"invalid session struct\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (rte_mempool_get(mempool, &sess_private_data)) {\n+\t\tAESNI_MB_LOG(ERR,\n+\t\t\t\t\"Couldn't get object from session mempool\");\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tret = aesni_mb_set_session_parameters(internals->mb_mgr,\n+\t\t\tsess_private_data, xform);\n+\tif (ret != 0) {\n+\t\tAESNI_MB_LOG(ERR, \"failed configure session parameters\");\n+\n+\t\t/* Return session to mempool */\n+\t\trte_mempool_put(mempool, sess_private_data);\n+\t\treturn ret;\n+\t}\n+\n+\tset_sym_session_private_data(sess, dev->driver_id,\n+\t\t\tsess_private_data);\n+\n+\treturn 0;\n+}\n+\n+/** Clear the memory of session so it doesn't leave key material behind */\n+static void\n+aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,\n+\t\tstruct rte_cryptodev_sym_session *sess)\n+{\n+\tuint8_t index = dev->driver_id;\n+\tvoid *sess_priv = get_sym_session_private_data(sess, index);\n+\n+\t/* Zero out the whole structure */\n+\tif (sess_priv) {\n+\t\tmemset(sess_priv, 0, sizeof(struct aesni_mb_session));\n+\t\tstruct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);\n+\t\tset_sym_session_private_data(sess, index, NULL);\n+\t\trte_mempool_put(sess_mp, sess_priv);\n+\t}\n+}\n+\n+struct rte_cryptodev_ops aesni_mb_pmd_ops = {\n+\t\t.dev_configure\t\t= aesni_mb_pmd_config,\n+\t\t.dev_start\t\t= aesni_mb_pmd_start,\n+\t\t.dev_stop\t\t= aesni_mb_pmd_stop,\n+\t\t.dev_close\t\t= aesni_mb_pmd_close,\n+\n+\t\t.stats_get\t\t= aesni_mb_pmd_stats_get,\n+\t\t.stats_reset\t\t= aesni_mb_pmd_stats_reset,\n+\n+\t\t.dev_infos_get\t\t= aesni_mb_pmd_info_get,\n+\n+\t\t.queue_pair_setup\t= aesni_mb_pmd_qp_setup,\n+\t\t.queue_pair_release\t= aesni_mb_pmd_qp_release,\n+\t\t.queue_pair_count\t= aesni_mb_pmd_qp_count,\n+\n+\t\t.sym_session_get_size\t= aesni_mb_pmd_sym_session_get_size,\n+\t\t.sym_session_configure\t= aesni_mb_pmd_sym_session_configure,\n+\t\t.sym_session_clear\t= aesni_mb_pmd_sym_session_clear\n+};\n+\n+struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;\ndiff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h\nindex d8021cdaa..d61abfe4f 100644\n--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h\n+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h\n@@ -5,7 +5,32 @@\n #ifndef _RTE_AESNI_MB_PMD_PRIVATE_H_\n #define _RTE_AESNI_MB_PMD_PRIVATE_H_\n \n+#include <intel-ipsec-mb.h>\n+\n+\n+/*\n+ * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,\n+ * so if macro is not defined, it means that the version is 0.49.\n+ */\n+#if !defined(IMB_VERSION_NUM)\n+#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))\n+#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)\n+#endif\n+\n+#if IMB_VERSION_NUM < IMB_VERSION(0, 52, 0)\n #include \"aesni_mb_ops.h\"\n+#endif\n+\n+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)\n+enum aesni_mb_vector_mode {\n+\tRTE_AESNI_MB_NOT_SUPPORTED = 0,\n+\tRTE_AESNI_MB_SSE,\n+\tRTE_AESNI_MB_AVX,\n+\tRTE_AESNI_MB_AVX2,\n+\tRTE_AESNI_MB_AVX512\n+};\n+#endif\n+\n \n #define CRYPTODEV_NAME_AESNI_MB_PMD\tcrypto_aesni_mb\n /**< AES-NI Multi buffer PMD device name */\n@@ -83,7 +108,9 @@ static const unsigned auth_digest_byte_lengths[] = {\n \t\t[AES_XCBC]\t= 16,\n \t\t[AES_CMAC]\t= 16,\n \t\t[AES_GMAC]\t= 12,\n-\t\t[NULL_HASH]\t\t= 0\n+\t\t[NULL_HASH]\t= 0,\n+\t/**< Vector mode dependent pointer table of the multi-buffer APIs */\n+\n };\n \n /**\n@@ -115,6 +142,10 @@ struct aesni_mb_private {\n \t/**< CPU vector instruction set mode */\n \tunsigned max_nb_queue_pairs;\n \t/**< Max number of queue pairs supported by device */\n+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)\n+\tMB_MGR *mb_mgr;\n+\t/**< Multi-buffer instance */\n+#endif\n };\n \n /** AESNI Multi buffer queue pair */\n@@ -122,13 +153,15 @@ struct aesni_mb_qp {\n \tuint16_t id;\n \t/**< Queue Pair Identifier */\n \tchar name[RTE_CRYPTODEV_NAME_MAX_LEN];\n+#if IMB_VERSION_NUM < IMB_VERSION(0, 52, 0)\n \t/**< Unique Queue Pair Name */\n \tconst struct aesni_mb_op_fns *op_fns;\n-\t/**< Vector mode dependent pointer table of the multi-buffer APIs */\n+#endif\n+\t/**< Unique Queue Pair Name */\n \tMB_MGR *mb_mgr;\n \t/**< Multi-buffer instance */\n \tstruct rte_ring *ingress_queue;\n-       /**< Ring for placing operations ready for processing */\n+\t/**< Ring for placing operations ready for processing */\n \tstruct rte_mempool *sess_mp;\n \t/**< Session Mempool */\n \tstruct rte_cryptodev_stats stats;\n@@ -153,7 +186,9 @@ struct aesni_mb_session {\n \t} iv;\n \t/**< IV parameters */\n \n-\t/** Cipher Parameters */\n+\t/** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;\n+\t/**< Vector mode dependent pointer table of the multi-buffer APIs */\n+\n \tstruct {\n \t\t/** Cipher direction - encrypt / decrypt */\n \t\tJOB_CIPHER_DIRECTION direction;\n@@ -234,14 +269,21 @@ struct aesni_mb_session {\n } __rte_cache_aligned;\n \n \n+\n+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)\n /**\n  *\n  */\n extern int\n+aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,\n+\t\tstruct aesni_mb_session *sess,\n+\t\tconst struct rte_crypto_sym_xform *xform);\n+#else\n+extern int\n aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,\n \t\tstruct aesni_mb_session *sess,\n \t\tconst struct rte_crypto_sym_xform *xform);\n-\n+#endif\n \n /** device specific operations function pointer structure */\n extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;\n",
    "prefixes": [
        "v3",
        "2/4"
    ]
}