Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 100010,
    "url": "http://patches.dpdk.org/api/patches/100010/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/20210929163035.608387-4-ciara.power@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20210929163035.608387-4-ciara.power@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20210929163035.608387-4-ciara.power@intel.com",
    "date": "2021-09-29T16:30:28",
    "name": "[v3,03/10] drivers/crypto: move aesni-mb PMD to IPsec-mb framework",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "bfc03ca3457b1ad3ac6192fd86e19f8b29da3168",
    "submitter": {
        "id": 978,
        "url": "http://patches.dpdk.org/api/people/978/?format=api",
        "name": "Power, Ciara",
        "email": "ciara.power@intel.com"
    },
    "delegate": {
        "id": 6690,
        "url": "http://patches.dpdk.org/api/users/6690/?format=api",
        "username": "akhil",
        "first_name": "akhil",
        "last_name": "goyal",
        "email": "gakhil@marvell.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/20210929163035.608387-4-ciara.power@intel.com/mbox/",
    "series": [
        {
            "id": 19269,
            "url": "http://patches.dpdk.org/api/series/19269/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=19269",
            "date": "2021-09-29T16:30:25",
            "name": "drivers/crypto: introduce ipsec_mb framework",
            "version": 3,
            "mbox": "http://patches.dpdk.org/series/19269/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/100010/comments/",
    "check": "success",
    "checks": "http://patches.dpdk.org/api/patches/100010/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 51334A0547;\n\tWed, 29 Sep 2021 18:31:02 +0200 (CEST)",
            "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 3CBF041103;\n\tWed, 29 Sep 2021 18:30:50 +0200 (CEST)",
            "from mga06.intel.com (mga06.intel.com [134.134.136.31])\n by mails.dpdk.org (Postfix) with ESMTP id 9B37E41103\n for <dev@dpdk.org>; Wed, 29 Sep 2021 18:30:47 +0200 (CEST)",
            "from orsmga002.jf.intel.com ([10.7.209.21])\n by orsmga104.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 29 Sep 2021 09:30:46 -0700",
            "from silpixa00400355.ir.intel.com (HELO\n silpixa00400355.ger.corp.intel.com) ([10.237.222.87])\n by orsmga002.jf.intel.com with ESMTP; 29 Sep 2021 09:30:44 -0700"
        ],
        "X-IronPort-AV": [
            "E=McAfee;i=\"6200,9189,10122\"; a=\"285997953\"",
            "E=Sophos;i=\"5.85,332,1624345200\"; d=\"scan'208\";a=\"285997953\"",
            "E=Sophos;i=\"5.85,332,1624345200\"; d=\"scan'208\";a=\"457092749\""
        ],
        "X-ExtLoop1": "1",
        "From": "Ciara Power <ciara.power@intel.com>",
        "To": "dev@dpdk.org",
        "Cc": "roy.fan.zhang@intel.com, piotrx.bronowski@intel.com, gakhil@marvell.com,\n\t=?utf-8?q?Ciara=C2=A0Power?= <ciara.power@intel.com>,\n Thomas Monjalon <thomas@monjalon.net>,\n Pablo de Lara <pablo.de.lara.guarch@intel.com>, Ray Kinsella <mdr@ashroe.eu>",
        "Date": "Wed, 29 Sep 2021 16:30:28 +0000",
        "Message-Id": "<20210929163035.608387-4-ciara.power@intel.com>",
        "X-Mailer": "git-send-email 2.25.1",
        "In-Reply-To": "<20210929163035.608387-1-ciara.power@intel.com>",
        "References": "<20210727083832.291687-1-roy.fan.zhang@intel.com>\n <20210929163035.608387-1-ciara.power@intel.com>",
        "MIME-Version": "1.0",
        "Content-Type": "text/plain; charset=UTF-8",
        "Content-Transfer-Encoding": "8bit",
        "Subject": "[dpdk-dev] [PATCH v3 03/10] drivers/crypto: move aesni-mb PMD to\n IPsec-mb framework",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "From: Piotr Bronowski <piotrx.bronowski@intel.com>\n\nThis patch removes the crypto/aesni_mb folder and gathers all\naesni-mb PMD implementation specific details into a single file,\npmd_aesni_mb.c in crypto/ipsec_mb.\n\nNow that intel-ipsec-mb v1.0 is the minimum supported version, old\nmacros can be replaced with the newer macros supported by this version.\n\nSigned-off-by: Piotr Bronowski <piotrx.bronowski@intel.com>\nSigned-off-by: Ciara Power <ciara.power@intel.com>\n\n---\nv3:\n  - Updated intel-ipsec-mb library macros.\n  - Fixed some formatting.\nv2: Updated maintainers file.\n---\n MAINTAINERS                                   |   10 +-\n doc/guides/cryptodevs/aesni_mb.rst            |    4 +-\n .../crypto/aesni_mb/aesni_mb_pmd_private.h    |  337 --\n drivers/crypto/aesni_mb/meson.build           |   25 -\n drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c    | 2232 ------------\n .../crypto/aesni_mb/rte_aesni_mb_pmd_ops.c    | 1126 -------\n drivers/crypto/aesni_mb/version.map           |    3 -\n drivers/crypto/ipsec_mb/meson.build           |    1 +\n drivers/crypto/ipsec_mb/pmd_aesni_mb.c        | 2977 +++++++++++++++++\n drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c    |    7 +-\n .../ipsec_mb/rte_ipsec_mb_pmd_private.h       |  143 +-\n drivers/crypto/meson.build                    |    1 -\n 12 files changed, 3131 insertions(+), 3735 deletions(-)\n delete mode 100644 drivers/crypto/aesni_mb/aesni_mb_pmd_private.h\n delete mode 100644 drivers/crypto/aesni_mb/meson.build\n delete mode 100644 drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c\n delete mode 100644 drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c\n delete mode 100644 drivers/crypto/aesni_mb/version.map\n create mode 100644 drivers/crypto/ipsec_mb/pmd_aesni_mb.c",
    "diff": "diff --git a/MAINTAINERS b/MAINTAINERS\nindex f1aaf7d408..7b00cd8791 100644\n--- a/MAINTAINERS\n+++ b/MAINTAINERS\n@@ -1049,13 +1049,6 @@ F: drivers/crypto/aesni_gcm/\n F: doc/guides/cryptodevs/aesni_gcm.rst\n F: doc/guides/cryptodevs/features/aesni_gcm.ini\n \n-Intel AES-NI Multi-Buffer\n-M: Declan Doherty <declan.doherty@intel.com>\n-M: Pablo de Lara <pablo.de.lara.guarch@intel.com>\n-F: drivers/crypto/aesni_mb/\n-F: doc/guides/cryptodevs/aesni_mb.rst\n-F: doc/guides/cryptodevs/features/aesni_mb.ini\n-\n Intel QuickAssist\n M: John Griffin <john.griffin@intel.com>\n M: Fiona Trahe <fiona.trahe@intel.com>\n@@ -1067,7 +1060,10 @@ F: doc/guides/cryptodevs/features/qat.ini\n \n IPsec MB\n M: Fan Zhang <roy.fan.zhang@intel.com>\n+M: Pablo de Lara <pablo.de.lara.guarch@intel.com>\n F: drivers/crypto/ipsec_mb/\n+F: doc/guides/cryptodevs/aesni_mb.rst\n+F: doc/guides/cryptodevs/features/aesni_mb.ini\n \n KASUMI\n M: Pablo de Lara <pablo.de.lara.guarch@intel.com>\ndiff --git a/doc/guides/cryptodevs/aesni_mb.rst b/doc/guides/cryptodevs/aesni_mb.rst\nindex a466d0ab48..3551a0dbd7 100644\n--- a/doc/guides/cryptodevs/aesni_mb.rst\n+++ b/doc/guides/cryptodevs/aesni_mb.rst\n@@ -130,7 +130,9 @@ and the Multi-Buffer library version supported by them:\n    18.02           0.48\n    18.05 - 19.02   0.49 - 0.52\n    19.05 - 19.08   0.52\n-   19.11+          0.52 - 1.0*\n+   19.11 - 20.08   0.52 - 0.55\n+   20.11 - 21.08   0.53 - 1.0*\n+   21.11+          1.0*\n    ==============  ============================\n \n \\* Multi-buffer library 1.0 or newer only works for Meson but not Make build system.\ndiff --git a/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h\ndeleted file mode 100644\nindex 11e7bf5d18..0000000000\n--- a/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h\n+++ /dev/null\n@@ -1,337 +0,0 @@\n-/* SPDX-License-Identifier: BSD-3-Clause\n- * Copyright(c) 2015-2016 Intel Corporation\n- */\n-\n-#ifndef _AESNI_MB_PMD_PRIVATE_H_\n-#define _AESNI_MB_PMD_PRIVATE_H_\n-\n-#include <intel-ipsec-mb.h>\n-\n-#if defined(RTE_LIB_SECURITY) && (IMB_VERSION_NUM) >= IMB_VERSION(0, 54, 0)\n-#define AESNI_MB_DOCSIS_SEC_ENABLED 1\n-#include <rte_security.h>\n-#include <rte_security_driver.h>\n-#endif\n-\n-enum aesni_mb_vector_mode {\n-\tRTE_AESNI_MB_NOT_SUPPORTED = 0,\n-\tRTE_AESNI_MB_SSE,\n-\tRTE_AESNI_MB_AVX,\n-\tRTE_AESNI_MB_AVX2,\n-\tRTE_AESNI_MB_AVX512\n-};\n-\n-#define CRYPTODEV_NAME_AESNI_MB_PMD\tcrypto_aesni_mb\n-/**< AES-NI Multi buffer PMD device name */\n-\n-/** AESNI_MB PMD LOGTYPE DRIVER */\n-extern int aesni_mb_logtype_driver;\n-\n-#define AESNI_MB_LOG(level, fmt, ...)  \\\n-\trte_log(RTE_LOG_ ## level, aesni_mb_logtype_driver,  \\\n-\t\t\t\"%s() line %u: \" fmt \"\\n\", __func__, __LINE__,  \\\n-\t\t\t\t\t## __VA_ARGS__)\n-\n-\n-#define HMAC_IPAD_VALUE\t\t\t(0x36)\n-#define HMAC_OPAD_VALUE\t\t\t(0x5C)\n-\n-/* Maximum length for digest */\n-#define DIGEST_LENGTH_MAX 64\n-static const unsigned auth_blocksize[] = {\n-\t\t[NULL_HASH]\t\t\t= 0,\n-\t\t[MD5]\t\t\t\t= 64,\n-\t\t[SHA1]\t\t\t\t= 64,\n-\t\t[SHA_224]\t\t\t= 64,\n-\t\t[SHA_256]\t\t\t= 64,\n-\t\t[SHA_384]\t\t\t= 128,\n-\t\t[SHA_512]\t\t\t= 128,\n-\t\t[AES_XCBC]\t\t\t= 16,\n-\t\t[AES_CCM]\t\t\t= 16,\n-\t\t[AES_CMAC]\t\t\t= 16,\n-\t\t[AES_GMAC]\t\t\t= 16,\n-\t\t[PLAIN_SHA1]\t\t\t= 64,\n-\t\t[PLAIN_SHA_224]\t\t\t= 64,\n-\t\t[PLAIN_SHA_256]\t\t\t= 64,\n-\t\t[PLAIN_SHA_384]\t\t\t= 128,\n-\t\t[PLAIN_SHA_512]\t\t\t= 128,\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\t\t[IMB_AUTH_ZUC_EIA3_BITLEN]\t= 16,\n-\t\t[IMB_AUTH_SNOW3G_UIA2_BITLEN]\t= 16,\n-\t\t[IMB_AUTH_KASUMI_UIA1]\t\t= 16\n-#endif\n-};\n-\n-/**\n- * Get the blocksize in bytes for a specified authentication algorithm\n- *\n- * @Note: this function will not return a valid value for a non-valid\n- * authentication algorithm\n- */\n-static inline unsigned\n-get_auth_algo_blocksize(JOB_HASH_ALG algo)\n-{\n-\treturn auth_blocksize[algo];\n-}\n-\n-static const unsigned auth_truncated_digest_byte_lengths[] = {\n-\t\t[MD5]\t\t\t\t= 12,\n-\t\t[SHA1]\t\t\t\t= 12,\n-\t\t[SHA_224]\t\t\t= 14,\n-\t\t[SHA_256]\t\t\t= 16,\n-\t\t[SHA_384]\t\t\t= 24,\n-\t\t[SHA_512]\t\t\t= 32,\n-\t\t[AES_XCBC]\t\t\t= 12,\n-\t\t[AES_CMAC]\t\t\t= 12,\n-\t\t[AES_CCM]\t\t\t= 8,\n-\t\t[NULL_HASH]\t\t\t= 0,\n-\t\t[AES_GMAC]\t\t\t= 12,\n-\t\t[PLAIN_SHA1]\t\t\t= 20,\n-\t\t[PLAIN_SHA_224]\t\t\t= 28,\n-\t\t[PLAIN_SHA_256]\t\t\t= 32,\n-\t\t[PLAIN_SHA_384]\t\t\t= 48,\n-\t\t[PLAIN_SHA_512]\t\t\t= 64,\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\t\t[IMB_AUTH_ZUC_EIA3_BITLEN]\t= 4,\n-\t\t[IMB_AUTH_SNOW3G_UIA2_BITLEN]\t= 4,\n-\t\t[IMB_AUTH_KASUMI_UIA1]\t\t= 4\n-#endif\n-};\n-\n-/**\n- * Get the IPsec specified truncated length in bytes of the HMAC digest for a\n- * specified authentication algorithm\n- *\n- * @Note: this function will not return a valid value for a non-valid\n- * authentication algorithm\n- */\n-static inline unsigned\n-get_truncated_digest_byte_length(JOB_HASH_ALG algo)\n-{\n-\treturn auth_truncated_digest_byte_lengths[algo];\n-}\n-\n-static const unsigned auth_digest_byte_lengths[] = {\n-\t\t[MD5]\t\t\t\t= 16,\n-\t\t[SHA1]\t\t\t\t= 20,\n-\t\t[SHA_224]\t\t\t= 28,\n-\t\t[SHA_256]\t\t\t= 32,\n-\t\t[SHA_384]\t\t\t= 48,\n-\t\t[SHA_512]\t\t\t= 64,\n-\t\t[AES_XCBC]\t\t\t= 16,\n-\t\t[AES_CMAC]\t\t\t= 16,\n-\t\t[AES_CCM]\t\t\t= 16,\n-\t\t[AES_GMAC]\t\t\t= 16,\n-\t\t[NULL_HASH]\t\t\t= 0,\n-\t\t[PLAIN_SHA1]\t\t\t= 20,\n-\t\t[PLAIN_SHA_224]\t\t\t= 28,\n-\t\t[PLAIN_SHA_256]\t\t\t= 32,\n-\t\t[PLAIN_SHA_384]\t\t\t= 48,\n-\t\t[PLAIN_SHA_512]\t\t\t= 64,\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\t\t[IMB_AUTH_ZUC_EIA3_BITLEN]\t= 4,\n-\t\t[IMB_AUTH_SNOW3G_UIA2_BITLEN]\t= 4,\n-\t\t[IMB_AUTH_KASUMI_UIA1]\t\t= 4\n-#endif\n-\t/**< Vector mode dependent pointer table of the multi-buffer APIs */\n-\n-};\n-\n-/**\n- * Get the full digest size in bytes for a specified authentication algorithm\n- * (if available in the Multi-buffer library)\n- *\n- * @Note: this function will not return a valid value for a non-valid\n- * authentication algorithm\n- */\n-static inline unsigned\n-get_digest_byte_length(JOB_HASH_ALG algo)\n-{\n-\treturn auth_digest_byte_lengths[algo];\n-}\n-\n-enum aesni_mb_operation {\n-\tAESNI_MB_OP_HASH_CIPHER,\n-\tAESNI_MB_OP_CIPHER_HASH,\n-\tAESNI_MB_OP_HASH_ONLY,\n-\tAESNI_MB_OP_CIPHER_ONLY,\n-\tAESNI_MB_OP_AEAD_HASH_CIPHER,\n-\tAESNI_MB_OP_AEAD_CIPHER_HASH,\n-\tAESNI_MB_OP_NOT_SUPPORTED\n-};\n-\n-/** private data structure for each virtual AESNI device */\n-struct aesni_mb_private {\n-\tenum aesni_mb_vector_mode vector_mode;\n-\t/**< CPU vector instruction set mode */\n-\tunsigned max_nb_queue_pairs;\n-\t/**< Max number of queue pairs supported by device */\n-\tMB_MGR *mb_mgr;\n-\t/**< Multi-buffer instance */\n-};\n-\n-/** AESNI Multi buffer queue pair */\n-struct aesni_mb_qp {\n-\tuint16_t id;\n-\t/**< Queue Pair Identifier */\n-\tchar name[RTE_CRYPTODEV_NAME_MAX_LEN];\n-\t/**< Unique Queue Pair Name */\n-\tMB_MGR *mb_mgr;\n-\t/**< Multi-buffer instance */\n-\tstruct rte_ring *ingress_queue;\n-\t/**< Ring for placing operations ready for processing */\n-\tstruct rte_mempool *sess_mp;\n-\t/**< Session Mempool */\n-\tstruct rte_mempool *sess_mp_priv;\n-\t/**< Session Private Data Mempool */\n-\tstruct rte_cryptodev_stats stats;\n-\t/**< Queue pair statistics */\n-\tuint8_t digest_idx;\n-\t/**< Index of the next slot to be used in temp_digests,\n-\t * to store the digest for a given operation\n-\t */\n-\tuint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];\n-\t/**< Buffers used to store the digest generated\n-\t * by the driver when verifying a digest provided\n-\t * by the user (using authentication verify operation)\n-\t */\n-} __rte_cache_aligned;\n-\n-/** AES-NI multi-buffer private session structure */\n-struct aesni_mb_session {\n-\tJOB_CHAIN_ORDER chain_order;\n-\tstruct {\n-\t\tuint16_t length;\n-\t\tuint16_t offset;\n-\t} iv;\n-\tstruct {\n-\t\tuint16_t length;\n-\t\tuint16_t offset;\n-\t} auth_iv;\n-\t/**< IV parameters */\n-\n-\t/** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;\n-\t/**< Vector mode dependent pointer table of the multi-buffer APIs */\n-\n-\tstruct {\n-\t\t/** Cipher direction - encrypt / decrypt */\n-\t\tJOB_CIPHER_DIRECTION direction;\n-\t\t/** Cipher mode - CBC / Counter */\n-\t\tJOB_CIPHER_MODE mode;\n-\n-\t\tuint64_t key_length_in_bytes;\n-\n-\t\tunion {\n-\t\t\tstruct {\n-\t\t\t\tuint32_t encode[60] __rte_aligned(16);\n-\t\t\t\t/**< encode key */\n-\t\t\t\tuint32_t decode[60] __rte_aligned(16);\n-\t\t\t\t/**< decode key */\n-\t\t\t} expanded_aes_keys;\n-\t\t\t/**< Expanded AES keys - Allocating space to\n-\t\t\t * contain the maximum expanded key size which\n-\t\t\t * is 240 bytes for 256 bit AES, calculate by:\n-\t\t\t * ((key size (bytes)) *\n-\t\t\t * ((number of rounds) + 1))\n-\t\t\t */\n-\t\t\tstruct {\n-\t\t\t\tconst void *ks_ptr[3];\n-\t\t\t\tuint64_t key[3][16];\n-\t\t\t} exp_3des_keys;\n-\t\t\t/**< Expanded 3DES keys */\n-\n-\t\t\tstruct gcm_key_data gcm_key;\n-\t\t\t/**< Expanded GCM key */\n-\t\t\tuint8_t zuc_cipher_key[16];\n-\t\t\t/**< ZUC cipher key */\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\t\t\tsnow3g_key_schedule_t pKeySched_snow3g_cipher;\n-\t\t\t/**< SNOW3G scheduled cipher key */\n-\t\t\tkasumi_key_sched_t pKeySched_kasumi_cipher;\n-\t\t\t/**< KASUMI scheduled cipher key */\n-#endif\n-\t\t};\n-\t} cipher;\n-\n-\t/** Authentication Parameters */\n-\tstruct {\n-\t\tJOB_HASH_ALG algo; /**< Authentication Algorithm */\n-\t\tenum rte_crypto_auth_operation operation;\n-\t\t/**< auth operation generate or verify */\n-\t\tunion {\n-\t\t\tstruct {\n-\t\t\t\tuint8_t inner[128] __rte_aligned(16);\n-\t\t\t\t/**< inner pad */\n-\t\t\t\tuint8_t outer[128] __rte_aligned(16);\n-\t\t\t\t/**< outer pad */\n-\t\t\t} pads;\n-\t\t\t/**< HMAC Authentication pads -\n-\t\t\t * allocating space for the maximum pad\n-\t\t\t * size supported which is 128 bytes for\n-\t\t\t * SHA512\n-\t\t\t */\n-\n-\t\t\tstruct {\n-\t\t\t    uint32_t k1_expanded[44] __rte_aligned(16);\n-\t\t\t    /**< k1 (expanded key). */\n-\t\t\t    uint8_t k2[16] __rte_aligned(16);\n-\t\t\t    /**< k2. */\n-\t\t\t    uint8_t k3[16] __rte_aligned(16);\n-\t\t\t    /**< k3. */\n-\t\t\t} xcbc;\n-\n-\t\t\tstruct {\n-\t\t\t\tuint32_t expkey[60] __rte_aligned(16);\n-\t\t\t\t\t\t    /**< k1 (expanded key). */\n-\t\t\t\tuint32_t skey1[4] __rte_aligned(16);\n-\t\t\t\t\t\t    /**< k2. */\n-\t\t\t\tuint32_t skey2[4] __rte_aligned(16);\n-\t\t\t\t\t\t    /**< k3. */\n-\t\t\t} cmac;\n-\t\t\t/**< Expanded XCBC authentication keys */\n-\t\t\tuint8_t zuc_auth_key[16];\n-\t\t\t/**< ZUC authentication key */\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\t\t\tsnow3g_key_schedule_t pKeySched_snow3g_auth;\n-\t\t\t/**< SNOW3G scheduled authentication key */\n-\t\t\tkasumi_key_sched_t pKeySched_kasumi_auth;\n-\t\t\t/**< KASUMI scheduled authentication key */\n-#endif\n-\t\t};\n-\t/** Generated digest size by the Multi-buffer library */\n-\tuint16_t gen_digest_len;\n-\t/** Requested digest size from Cryptodev */\n-\tuint16_t req_digest_len;\n-\n-\t} auth;\n-\tstruct {\n-\t\t/** AAD data length */\n-\t\tuint16_t aad_len;\n-\t} aead;\n-} __rte_cache_aligned;\n-\n-extern int\n-aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,\n-\t\tstruct aesni_mb_session *sess,\n-\t\tconst struct rte_crypto_sym_xform *xform);\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-extern int\n-aesni_mb_set_docsis_sec_session_parameters(\n-\t\t__rte_unused struct rte_cryptodev *dev,\n-\t\tstruct rte_security_session_conf *conf,\n-\t\tvoid *sess);\n-#endif\n-\n-/** device specific operations function pointer structures */\n-extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-extern struct rte_security_ops *rte_aesni_mb_pmd_sec_ops;\n-#endif\n-\n-extern uint32_t\n-aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,\n-\tstruct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,\n-\tstruct rte_crypto_sym_vec *vec);\n-\n-#endif /* _AESNI_MB_PMD_PRIVATE_H_ */\ndiff --git a/drivers/crypto/aesni_mb/meson.build b/drivers/crypto/aesni_mb/meson.build\ndeleted file mode 100644\nindex ed6b9f53e4..0000000000\n--- a/drivers/crypto/aesni_mb/meson.build\n+++ /dev/null\n@@ -1,25 +0,0 @@\n-# SPDX-License-Identifier: BSD-3-Clause\n-# Copyright(c) 2018 Intel Corporation\n-\n-IMB_required_ver = '0.52.0'\n-lib = cc.find_library('IPSec_MB', required: false)\n-if not lib.found()\n-    build = false\n-    reason = 'missing dependency, \"libIPSec_MB\"'\n-else\n-    ext_deps += lib\n-\n-    # version comes with quotes, so we split based on \" and take the middle\n-    imb_ver = cc.get_define('IMB_VERSION_STR',\n-        prefix : '#include<intel-ipsec-mb.h>').split('\"')[1]\n-\n-    if (imb_ver == '') or (imb_ver.version_compare('<' + IMB_required_ver))\n-        reason = 'IPSec_MB version >= @0@ is required, found version @1@'.format(\n-                IMB_required_ver, imb_ver)\n-        build = false\n-    endif\n-\n-endif\n-\n-sources = files('rte_aesni_mb_pmd.c', 'rte_aesni_mb_pmd_ops.c')\n-deps += ['bus_vdev', 'net', 'security']\ndiff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c\ndeleted file mode 100644\nindex 60963a8208..0000000000\n--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c\n+++ /dev/null\n@@ -1,2232 +0,0 @@\n-/* SPDX-License-Identifier: BSD-3-Clause\n- * Copyright(c) 2015-2017 Intel Corporation\n- */\n-\n-#include <intel-ipsec-mb.h>\n-\n-#include <rte_common.h>\n-#include <rte_hexdump.h>\n-#include <rte_cryptodev.h>\n-#include <cryptodev_pmd.h>\n-#include <rte_bus_vdev.h>\n-#include <rte_malloc.h>\n-#include <rte_cpuflags.h>\n-#include <rte_per_lcore.h>\n-#include <rte_ether.h>\n-\n-#include \"aesni_mb_pmd_private.h\"\n-\n-#define AES_CCM_DIGEST_MIN_LEN 4\n-#define AES_CCM_DIGEST_MAX_LEN 16\n-#define HMAC_MAX_BLOCK_SIZE 128\n-static uint8_t cryptodev_driver_id;\n-\n-/*\n- * Needed to support CPU-CRYPTO API (rte_cryptodev_sym_cpu_crypto_process),\n- * as we still use JOB based API even for synchronous processing.\n- */\n-static RTE_DEFINE_PER_LCORE(MB_MGR *, sync_mb_mgr);\n-\n-typedef void (*hash_one_block_t)(const void *data, void *digest);\n-typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);\n-\n-/**\n- * Calculate the authentication pre-computes\n- *\n- * @param one_block_hash\tFunction pointer to calculate digest on ipad/opad\n- * @param ipad\t\t\tInner pad output byte array\n- * @param opad\t\t\tOuter pad output byte array\n- * @param hkey\t\t\tAuthentication key\n- * @param hkey_len\t\tAuthentication key length\n- * @param blocksize\t\tBlock size of selected hash algo\n- */\n-static void\n-calculate_auth_precomputes(hash_one_block_t one_block_hash,\n-\t\tuint8_t *ipad, uint8_t *opad,\n-\t\tconst uint8_t *hkey, uint16_t hkey_len,\n-\t\tuint16_t blocksize)\n-{\n-\tunsigned i, length;\n-\n-\tuint8_t ipad_buf[blocksize] __rte_aligned(16);\n-\tuint8_t opad_buf[blocksize] __rte_aligned(16);\n-\n-\t/* Setup inner and outer pads */\n-\tmemset(ipad_buf, HMAC_IPAD_VALUE, blocksize);\n-\tmemset(opad_buf, HMAC_OPAD_VALUE, blocksize);\n-\n-\t/* XOR hash key with inner and outer pads */\n-\tlength = hkey_len > blocksize ? blocksize : hkey_len;\n-\n-\tfor (i = 0; i < length; i++) {\n-\t\tipad_buf[i] ^= hkey[i];\n-\t\topad_buf[i] ^= hkey[i];\n-\t}\n-\n-\t/* Compute partial hashes */\n-\t(*one_block_hash)(ipad_buf, ipad);\n-\t(*one_block_hash)(opad_buf, opad);\n-\n-\t/* Clean up stack */\n-\tmemset(ipad_buf, 0, blocksize);\n-\tmemset(opad_buf, 0, blocksize);\n-}\n-\n-/** Get xform chain order */\n-static enum aesni_mb_operation\n-aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)\n-{\n-\tif (xform == NULL)\n-\t\treturn AESNI_MB_OP_NOT_SUPPORTED;\n-\n-\tif (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {\n-\t\tif (xform->next == NULL)\n-\t\t\treturn AESNI_MB_OP_CIPHER_ONLY;\n-\t\tif (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)\n-\t\t\treturn AESNI_MB_OP_CIPHER_HASH;\n-\t}\n-\n-\tif (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {\n-\t\tif (xform->next == NULL)\n-\t\t\treturn AESNI_MB_OP_HASH_ONLY;\n-\t\tif (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)\n-\t\t\treturn AESNI_MB_OP_HASH_CIPHER;\n-\t}\n-#if IMB_VERSION_NUM > IMB_VERSION(0, 52, 0)\n-\tif (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {\n-\t\tif (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {\n-\t\t\t/*\n-\t\t\t * CCM requires to hash first and cipher later\n-\t\t\t * when encrypting\n-\t\t\t */\n-\t\t\tif (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)\n-\t\t\t\treturn AESNI_MB_OP_AEAD_HASH_CIPHER;\n-\t\t\telse\n-\t\t\t\treturn AESNI_MB_OP_AEAD_CIPHER_HASH;\n-\t\t} else {\n-\t\t\tif (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)\n-\t\t\t\treturn AESNI_MB_OP_AEAD_CIPHER_HASH;\n-\t\t\telse\n-\t\t\t\treturn AESNI_MB_OP_AEAD_HASH_CIPHER;\n-\t\t}\n-\t}\n-#else\n-\tif (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {\n-\t\tif (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||\n-\t\t\t\txform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {\n-\t\t\tif (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)\n-\t\t\t\treturn AESNI_MB_OP_AEAD_CIPHER_HASH;\n-\t\t\telse\n-\t\t\t\treturn AESNI_MB_OP_AEAD_HASH_CIPHER;\n-\t\t}\n-\t}\n-#endif\n-\n-\treturn AESNI_MB_OP_NOT_SUPPORTED;\n-}\n-\n-static inline int\n-is_aead_algo(JOB_HASH_ALG hash_alg, JOB_CIPHER_MODE cipher_mode)\n-{\n-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM\n-\treturn (hash_alg == IMB_AUTH_CHACHA20_POLY1305 || hash_alg == AES_CCM ||\n-\t\t(hash_alg == AES_GMAC && cipher_mode == GCM));\n-#else\n-\treturn ((hash_alg == AES_GMAC && cipher_mode == GCM) ||\n-\t\thash_alg == AES_CCM);\n-#endif\n-}\n-\n-/** Set session authentication parameters */\n-static int\n-aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,\n-\t\tstruct aesni_mb_session *sess,\n-\t\tconst struct rte_crypto_sym_xform *xform)\n-{\n-\thash_one_block_t hash_oneblock_fn = NULL;\n-\tunsigned int key_larger_block_size = 0;\n-\tuint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };\n-\tuint32_t auth_precompute = 1;\n-\n-\tif (xform == NULL) {\n-\t\tsess->auth.algo = NULL_HASH;\n-\t\treturn 0;\n-\t}\n-\n-\tif (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {\n-\t\tAESNI_MB_LOG(ERR, \"Crypto xform struct not of type auth\");\n-\t\treturn -1;\n-\t}\n-\n-\t/* Set IV parameters */\n-\tsess->auth_iv.offset = xform->auth.iv.offset;\n-\tsess->auth_iv.length = xform->auth.iv.length;\n-\n-\t/* Set the request digest size */\n-\tsess->auth.req_digest_len = xform->auth.digest_length;\n-\n-\t/* Select auth generate/verify */\n-\tsess->auth.operation = xform->auth.op;\n-\n-\t/* Set Authentication Parameters */\n-\tif (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {\n-\t\tsess->auth.algo = AES_XCBC;\n-\n-\t\tuint16_t xcbc_mac_digest_len =\n-\t\t\tget_truncated_digest_byte_length(AES_XCBC);\n-\t\tif (sess->auth.req_digest_len != xcbc_mac_digest_len) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n-\n-\t\tIMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,\n-\t\t\t\tsess->auth.xcbc.k1_expanded,\n-\t\t\t\tsess->auth.xcbc.k2, sess->auth.xcbc.k3);\n-\t\treturn 0;\n-\t}\n-\n-\tif (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {\n-\t\tuint32_t dust[4*15];\n-\n-\t\tsess->auth.algo = AES_CMAC;\n-\n-\t\tuint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);\n-\n-\t\tif (sess->auth.req_digest_len > cmac_digest_len) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\t/*\n-\t\t * Multi-buffer lib supports digest sizes from 4 to 16 bytes\n-\t\t * in version 0.50 and sizes of 12 and 16 bytes,\n-\t\t * in version 0.49.\n-\t\t * If size requested is different, generate the full digest\n-\t\t * (16 bytes) in a temporary location and then memcpy\n-\t\t * the requested number of bytes.\n-\t\t */\n-\t\tif (sess->auth.req_digest_len < 4)\n-\t\t\tsess->auth.gen_digest_len = cmac_digest_len;\n-\t\telse\n-\t\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n-\n-\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,\n-\t\t\t\tsess->auth.cmac.expkey, dust);\n-\t\tIMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,\n-\t\t\t\tsess->auth.cmac.skey1, sess->auth.cmac.skey2);\n-\t\treturn 0;\n-\t}\n-\n-\tif (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {\n-\t\tif (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {\n-\t\t\tsess->cipher.direction = ENCRYPT;\n-\t\t\tsess->chain_order = CIPHER_HASH;\n-\t\t} else\n-\t\t\tsess->cipher.direction = DECRYPT;\n-\n-\t\tsess->auth.algo = AES_GMAC;\n-\t\tif (sess->auth.req_digest_len > get_digest_byte_length(AES_GMAC)) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n-\t\tsess->iv.length = xform->auth.iv.length;\n-\t\tsess->iv.offset = xform->auth.iv.offset;\n-\n-\t\tswitch (xform->auth.key.length) {\n-\t\tcase AES_128_BYTES:\n-\t\t\tIMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,\n-\t\t\t\t&sess->cipher.gcm_key);\n-\t\t\tsess->cipher.key_length_in_bytes = AES_128_BYTES;\n-\t\t\tbreak;\n-\t\tcase AES_192_BYTES:\n-\t\t\tIMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,\n-\t\t\t\t&sess->cipher.gcm_key);\n-\t\t\tsess->cipher.key_length_in_bytes = AES_192_BYTES;\n-\t\t\tbreak;\n-\t\tcase AES_256_BYTES:\n-\t\t\tIMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,\n-\t\t\t\t&sess->cipher.gcm_key);\n-\t\t\tsess->cipher.key_length_in_bytes = AES_256_BYTES;\n-\t\t\tbreak;\n-\t\tdefault:\n-\t\t\tRTE_LOG(ERR, PMD, \"failed to parse test type\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\n-\t\treturn 0;\n-\t}\n-\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\tif (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {\n-\t\tsess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;\n-\t\tuint16_t zuc_eia3_digest_len =\n-\t\t\tget_truncated_digest_byte_length(IMB_AUTH_ZUC_EIA3_BITLEN);\n-\t\tif (sess->auth.req_digest_len != zuc_eia3_digest_len) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n-\n-\t\tmemcpy(sess->auth.zuc_auth_key, xform->auth.key.data, 16);\n-\t\treturn 0;\n-\t} else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {\n-\t\tsess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN;\n-\t\tuint16_t snow3g_uia2_digest_len =\n-\t\t\tget_truncated_digest_byte_length(IMB_AUTH_SNOW3G_UIA2_BITLEN);\n-\t\tif (sess->auth.req_digest_len != snow3g_uia2_digest_len) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n-\n-\t\tIMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->auth.key.data,\n-\t\t\t\t\t&sess->auth.pKeySched_snow3g_auth);\n-\t\treturn 0;\n-\t} else if (xform->auth.algo == RTE_CRYPTO_AUTH_KASUMI_F9) {\n-\t\tsess->auth.algo = IMB_AUTH_KASUMI_UIA1;\n-\t\tuint16_t kasumi_f9_digest_len =\n-\t\t\tget_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1);\n-\t\tif (sess->auth.req_digest_len != kasumi_f9_digest_len) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n-\n-\t\tIMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr, xform->auth.key.data,\n-\t\t\t\t\t&sess->auth.pKeySched_kasumi_auth);\n-\t\treturn 0;\n-\t}\n-#endif\n-\n-\tswitch (xform->auth.algo) {\n-\tcase RTE_CRYPTO_AUTH_MD5_HMAC:\n-\t\tsess->auth.algo = MD5;\n-\t\thash_oneblock_fn = mb_mgr->md5_one_block;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA1_HMAC:\n-\t\tsess->auth.algo = SHA1;\n-\t\thash_oneblock_fn = mb_mgr->sha1_one_block;\n-\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {\n-\t\t\tIMB_SHA1(mb_mgr,\n-\t\t\t\txform->auth.key.data,\n-\t\t\t\txform->auth.key.length,\n-\t\t\t\thashed_key);\n-\t\t\tkey_larger_block_size = 1;\n-\t\t}\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA1:\n-\t\tsess->auth.algo = PLAIN_SHA1;\n-\t\tauth_precompute = 0;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA224_HMAC:\n-\t\tsess->auth.algo = SHA_224;\n-\t\thash_oneblock_fn = mb_mgr->sha224_one_block;\n-\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {\n-\t\t\tIMB_SHA224(mb_mgr,\n-\t\t\t\txform->auth.key.data,\n-\t\t\t\txform->auth.key.length,\n-\t\t\t\thashed_key);\n-\t\t\tkey_larger_block_size = 1;\n-\t\t}\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA224:\n-\t\tsess->auth.algo = PLAIN_SHA_224;\n-\t\tauth_precompute = 0;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA256_HMAC:\n-\t\tsess->auth.algo = SHA_256;\n-\t\thash_oneblock_fn = mb_mgr->sha256_one_block;\n-\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {\n-\t\t\tIMB_SHA256(mb_mgr,\n-\t\t\t\txform->auth.key.data,\n-\t\t\t\txform->auth.key.length,\n-\t\t\t\thashed_key);\n-\t\t\tkey_larger_block_size = 1;\n-\t\t}\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA256:\n-\t\tsess->auth.algo = PLAIN_SHA_256;\n-\t\tauth_precompute = 0;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA384_HMAC:\n-\t\tsess->auth.algo = SHA_384;\n-\t\thash_oneblock_fn = mb_mgr->sha384_one_block;\n-\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {\n-\t\t\tIMB_SHA384(mb_mgr,\n-\t\t\t\txform->auth.key.data,\n-\t\t\t\txform->auth.key.length,\n-\t\t\t\thashed_key);\n-\t\t\tkey_larger_block_size = 1;\n-\t\t}\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA384:\n-\t\tsess->auth.algo = PLAIN_SHA_384;\n-\t\tauth_precompute = 0;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA512_HMAC:\n-\t\tsess->auth.algo = SHA_512;\n-\t\thash_oneblock_fn = mb_mgr->sha512_one_block;\n-\t\tif (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {\n-\t\t\tIMB_SHA512(mb_mgr,\n-\t\t\t\txform->auth.key.data,\n-\t\t\t\txform->auth.key.length,\n-\t\t\t\thashed_key);\n-\t\t\tkey_larger_block_size = 1;\n-\t\t}\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AUTH_SHA512:\n-\t\tsess->auth.algo = PLAIN_SHA_512;\n-\t\tauth_precompute = 0;\n-\t\tbreak;\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported authentication algorithm selection\");\n-\t\treturn -ENOTSUP;\n-\t}\n-\tuint16_t trunc_digest_size =\n-\t\t\tget_truncated_digest_byte_length(sess->auth.algo);\n-\tuint16_t full_digest_size =\n-\t\t\tget_digest_byte_length(sess->auth.algo);\n-\n-\tif (sess->auth.req_digest_len > full_digest_size ||\n-\t\t\tsess->auth.req_digest_len == 0) {\n-\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\tif (sess->auth.req_digest_len != trunc_digest_size &&\n-\t\t\tsess->auth.req_digest_len != full_digest_size)\n-\t\tsess->auth.gen_digest_len = full_digest_size;\n-\telse\n-\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n-\n-\t/* Plain SHA does not require precompute key */\n-\tif (auth_precompute == 0)\n-\t\treturn 0;\n-\n-\t/* Calculate Authentication precomputes */\n-\tif (key_larger_block_size) {\n-\t\tcalculate_auth_precomputes(hash_oneblock_fn,\n-\t\t\tsess->auth.pads.inner, sess->auth.pads.outer,\n-\t\t\thashed_key,\n-\t\t\txform->auth.key.length,\n-\t\t\tget_auth_algo_blocksize(sess->auth.algo));\n-\t} else {\n-\t\tcalculate_auth_precomputes(hash_oneblock_fn,\n-\t\t\tsess->auth.pads.inner, sess->auth.pads.outer,\n-\t\t\txform->auth.key.data,\n-\t\t\txform->auth.key.length,\n-\t\t\tget_auth_algo_blocksize(sess->auth.algo));\n-\t}\n-\n-\treturn 0;\n-}\n-\n-/** Set session cipher parameters */\n-static int\n-aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,\n-\t\tstruct aesni_mb_session *sess,\n-\t\tconst struct rte_crypto_sym_xform *xform)\n-{\n-\tuint8_t is_aes = 0;\n-\tuint8_t is_3DES = 0;\n-\tuint8_t is_docsis = 0;\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\tuint8_t is_zuc = 0;\n-\tuint8_t is_snow3g = 0;\n-\tuint8_t is_kasumi = 0;\n-#endif\n-\n-\tif (xform == NULL) {\n-\t\tsess->cipher.mode = NULL_CIPHER;\n-\t\treturn 0;\n-\t}\n-\n-\tif (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {\n-\t\tAESNI_MB_LOG(ERR, \"Crypto xform struct not of type cipher\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\t/* Select cipher direction */\n-\tswitch (xform->cipher.op) {\n-\tcase RTE_CRYPTO_CIPHER_OP_ENCRYPT:\n-\t\tsess->cipher.direction = ENCRYPT;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_CIPHER_OP_DECRYPT:\n-\t\tsess->cipher.direction = DECRYPT;\n-\t\tbreak;\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Invalid cipher operation parameter\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\t/* Select cipher mode */\n-\tswitch (xform->cipher.algo) {\n-\tcase RTE_CRYPTO_CIPHER_AES_CBC:\n-\t\tsess->cipher.mode = CBC;\n-\t\tis_aes = 1;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_CIPHER_AES_CTR:\n-\t\tsess->cipher.mode = CNTR;\n-\t\tis_aes = 1;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_CIPHER_AES_DOCSISBPI:\n-\t\tsess->cipher.mode = DOCSIS_SEC_BPI;\n-\t\tis_docsis = 1;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_CIPHER_DES_CBC:\n-\t\tsess->cipher.mode = DES;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_CIPHER_DES_DOCSISBPI:\n-\t\tsess->cipher.mode = DOCSIS_DES;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_CIPHER_3DES_CBC:\n-\t\tsess->cipher.mode = DES3;\n-\t\tis_3DES = 1;\n-\t\tbreak;\n-#if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM\n-\tcase RTE_CRYPTO_CIPHER_AES_ECB:\n-\t\tsess->cipher.mode = ECB;\n-\t\tis_aes = 1;\n-\t\tbreak;\n-#endif\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\tcase RTE_CRYPTO_CIPHER_ZUC_EEA3:\n-\t\tsess->cipher.mode = IMB_CIPHER_ZUC_EEA3;\n-\t\tis_zuc = 1;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_CIPHER_SNOW3G_UEA2:\n-\t\tsess->cipher.mode = IMB_CIPHER_SNOW3G_UEA2_BITLEN;\n-\t\tis_snow3g = 1;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_CIPHER_KASUMI_F8:\n-\t\tsess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN;\n-\t\tis_kasumi = 1;\n-\t\tbreak;\n-#endif\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported cipher mode parameter\");\n-\t\treturn -ENOTSUP;\n-\t}\n-\n-\t/* Set IV parameters */\n-\tsess->iv.offset = xform->cipher.iv.offset;\n-\tsess->iv.length = xform->cipher.iv.length;\n-\n-\t/* Check key length and choose key expansion function for AES */\n-\tif (is_aes) {\n-\t\tswitch (xform->cipher.key.length) {\n-\t\tcase AES_128_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_128_BYTES;\n-\t\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n-\t\t\tbreak;\n-\t\tcase AES_192_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_192_BYTES;\n-\t\t\tIMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n-\t\t\tbreak;\n-\t\tcase AES_256_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_256_BYTES;\n-\t\t\tIMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n-\t\t\tbreak;\n-\t\tdefault:\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t} else if (is_docsis) {\n-\t\tswitch (xform->cipher.key.length) {\n-\t\tcase AES_128_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_128_BYTES;\n-\t\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n-\t\t\tbreak;\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\t\tcase AES_256_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_256_BYTES;\n-\t\t\tIMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n-\t\t\tbreak;\n-#endif\n-\t\tdefault:\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t} else if (is_3DES) {\n-\t\tuint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],\n-\t\t\t\tsess->cipher.exp_3des_keys.key[1],\n-\t\t\t\tsess->cipher.exp_3des_keys.key[2]};\n-\n-\t\tswitch (xform->cipher.key.length) {\n-\t\tcase  24:\n-\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n-\t\t\t\t\txform->cipher.key.data);\n-\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[1],\n-\t\t\t\t\txform->cipher.key.data + 8);\n-\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[2],\n-\t\t\t\t\txform->cipher.key.data + 16);\n-\n-\t\t\t/* Initialize keys - 24 bytes: [K1-K2-K3] */\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];\n-\t\t\tbreak;\n-\t\tcase 16:\n-\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n-\t\t\t\t\txform->cipher.key.data);\n-\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[1],\n-\t\t\t\t\txform->cipher.key.data + 8);\n-\t\t\t/* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];\n-\t\t\tbreak;\n-\t\tcase 8:\n-\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n-\t\t\t\t\txform->cipher.key.data);\n-\n-\t\t\t/* Initialize keys - 8 bytes: [K1 = K2 = K3] */\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];\n-\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];\n-\t\t\tbreak;\n-\t\tdefault:\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\n-\t\tsess->cipher.key_length_in_bytes = 24;\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\t} else if (is_zuc) {\n-\t\tif (xform->cipher.key.length != 16) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->cipher.key_length_in_bytes = 16;\n-\t\tmemcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data,\n-\t\t\t16);\n-\t} else if (is_snow3g) {\n-\t\tif (xform->cipher.key.length != 16) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->cipher.key_length_in_bytes = 16;\n-\t\tIMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->cipher.key.data,\n-\t\t\t\t\t&sess->cipher.pKeySched_snow3g_cipher);\n-\t} else if (is_kasumi) {\n-\t\tif (xform->cipher.key.length != 16) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->cipher.key_length_in_bytes = 16;\n-\t\tIMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data,\n-\t\t\t\t\t&sess->cipher.pKeySched_kasumi_cipher);\n-#endif\n-\t} else {\n-\t\tif (xform->cipher.key.length != 8) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->cipher.key_length_in_bytes = 8;\n-\n-\t\tIMB_DES_KEYSCHED(mb_mgr,\n-\t\t\t(uint64_t *)sess->cipher.expanded_aes_keys.encode,\n-\t\t\t\txform->cipher.key.data);\n-\t\tIMB_DES_KEYSCHED(mb_mgr,\n-\t\t\t(uint64_t *)sess->cipher.expanded_aes_keys.decode,\n-\t\t\t\txform->cipher.key.data);\n-\t}\n-\n-\treturn 0;\n-}\n-\n-static int\n-aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,\n-\t\tstruct aesni_mb_session *sess,\n-\t\tconst struct rte_crypto_sym_xform *xform)\n-{\n-\tswitch (xform->aead.op) {\n-\tcase RTE_CRYPTO_AEAD_OP_ENCRYPT:\n-\t\tsess->cipher.direction = ENCRYPT;\n-\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;\n-\t\tbreak;\n-\tcase RTE_CRYPTO_AEAD_OP_DECRYPT:\n-\t\tsess->cipher.direction = DECRYPT;\n-\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;\n-\t\tbreak;\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Invalid aead operation parameter\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\t/* Set IV parameters */\n-\tsess->iv.offset = xform->aead.iv.offset;\n-\tsess->iv.length = xform->aead.iv.length;\n-\n-\t/* Set digest sizes */\n-\tsess->auth.req_digest_len = xform->aead.digest_length;\n-\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n-\n-\tswitch (xform->aead.algo) {\n-\tcase RTE_CRYPTO_AEAD_AES_CCM:\n-\t\tsess->cipher.mode = CCM;\n-\t\tsess->auth.algo = AES_CCM;\n-\n-\t\t/* Check key length and choose key expansion function for AES */\n-\t\tswitch (xform->aead.key.length) {\n-\t\tcase AES_128_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_128_BYTES;\n-\t\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n-\t\t\tbreak;\n-\t\tcase AES_256_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_256_BYTES;\n-\t\t\tIMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n-\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n-\t\t\tbreak;\n-\t\tdefault:\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\n-\t\t/* CCM digests must be between 4 and 16 and an even number */\n-\t\tif (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||\n-\t\t\t\tsess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||\n-\t\t\t\t(sess->auth.req_digest_len & 1) == 1) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tbreak;\n-\n-\tcase RTE_CRYPTO_AEAD_AES_GCM:\n-\t\tsess->cipher.mode = GCM;\n-\t\tsess->auth.algo = AES_GMAC;\n-\n-\t\tswitch (xform->aead.key.length) {\n-\t\tcase AES_128_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_128_BYTES;\n-\t\t\tIMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,\n-\t\t\t\t&sess->cipher.gcm_key);\n-\t\t\tbreak;\n-\t\tcase AES_192_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_192_BYTES;\n-\t\t\tIMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,\n-\t\t\t\t&sess->cipher.gcm_key);\n-\t\t\tbreak;\n-\t\tcase AES_256_BYTES:\n-\t\t\tsess->cipher.key_length_in_bytes = AES_256_BYTES;\n-\t\t\tIMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,\n-\t\t\t\t&sess->cipher.gcm_key);\n-\t\t\tbreak;\n-\t\tdefault:\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid cipher key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\n-\t\t/* GCM digest size must be between 1 and 16 */\n-\t\tif (sess->auth.req_digest_len == 0 ||\n-\t\t\t\tsess->auth.req_digest_len > 16) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tbreak;\n-\n-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM\n-\tcase RTE_CRYPTO_AEAD_CHACHA20_POLY1305:\n-\t\tsess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305;\n-\t\tsess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;\n-\n-\t\tif (xform->aead.key.length != 32) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid key length\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tsess->cipher.key_length_in_bytes = 32;\n-\t\tmemcpy(sess->cipher.expanded_aes_keys.encode,\n-\t\t\txform->aead.key.data, 32);\n-\t\tif (sess->auth.req_digest_len != 16) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid digest size\\n\");\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\t\tbreak;\n-#endif\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported aead mode parameter\");\n-\t\treturn -ENOTSUP;\n-\t}\n-\n-\treturn 0;\n-}\n-\n-/** Parse crypto xform chain and set private session parameters */\n-int\n-aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,\n-\t\tstruct aesni_mb_session *sess,\n-\t\tconst struct rte_crypto_sym_xform *xform)\n-{\n-\tconst struct rte_crypto_sym_xform *auth_xform = NULL;\n-\tconst struct rte_crypto_sym_xform *cipher_xform = NULL;\n-\tconst struct rte_crypto_sym_xform *aead_xform = NULL;\n-\tint ret;\n-\n-\t/* Select Crypto operation - hash then cipher / cipher then hash */\n-\tswitch (aesni_mb_get_chain_order(xform)) {\n-\tcase AESNI_MB_OP_HASH_CIPHER:\n-\t\tsess->chain_order = HASH_CIPHER;\n-\t\tauth_xform = xform;\n-\t\tcipher_xform = xform->next;\n-\t\tbreak;\n-\tcase AESNI_MB_OP_CIPHER_HASH:\n-\t\tsess->chain_order = CIPHER_HASH;\n-\t\tauth_xform = xform->next;\n-\t\tcipher_xform = xform;\n-\t\tbreak;\n-\tcase AESNI_MB_OP_HASH_ONLY:\n-\t\tsess->chain_order = HASH_CIPHER;\n-\t\tauth_xform = xform;\n-\t\tcipher_xform = NULL;\n-\t\tbreak;\n-\tcase AESNI_MB_OP_CIPHER_ONLY:\n-\t\t/*\n-\t\t * Multi buffer library operates only at two modes,\n-\t\t * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,\n-\t\t * chain order depends on cipher operation: encryption is always\n-\t\t * the first operation and decryption the last one.\n-\t\t */\n-\t\tif (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)\n-\t\t\tsess->chain_order = CIPHER_HASH;\n-\t\telse\n-\t\t\tsess->chain_order = HASH_CIPHER;\n-\t\tauth_xform = NULL;\n-\t\tcipher_xform = xform;\n-\t\tbreak;\n-\tcase AESNI_MB_OP_AEAD_CIPHER_HASH:\n-\t\tsess->chain_order = CIPHER_HASH;\n-\t\tsess->aead.aad_len = xform->aead.aad_length;\n-\t\taead_xform = xform;\n-\t\tbreak;\n-\tcase AESNI_MB_OP_AEAD_HASH_CIPHER:\n-\t\tsess->chain_order = HASH_CIPHER;\n-\t\tsess->aead.aad_len = xform->aead.aad_length;\n-\t\taead_xform = xform;\n-\t\tbreak;\n-\tcase AESNI_MB_OP_NOT_SUPPORTED:\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported operation chain order parameter\");\n-\t\treturn -ENOTSUP;\n-\t}\n-\n-\t/* Default IV length = 0 */\n-\tsess->iv.length = 0;\n-\tsess->auth_iv.length = 0;\n-\n-\tret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);\n-\tif (ret != 0) {\n-\t\tAESNI_MB_LOG(ERR, \"Invalid/unsupported authentication parameters\");\n-\t\treturn ret;\n-\t}\n-\n-\tret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,\n-\t\t\tcipher_xform);\n-\tif (ret != 0) {\n-\t\tAESNI_MB_LOG(ERR, \"Invalid/unsupported cipher parameters\");\n-\t\treturn ret;\n-\t}\n-\n-\tif (aead_xform) {\n-\t\tret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,\n-\t\t\t\taead_xform);\n-\t\tif (ret != 0) {\n-\t\t\tAESNI_MB_LOG(ERR, \"Invalid/unsupported aead parameters\");\n-\t\t\treturn ret;\n-\t\t}\n-\t}\n-\n-\treturn 0;\n-}\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-/** Check DOCSIS security session configuration is valid */\n-static int\n-check_docsis_sec_session(struct rte_security_session_conf *conf)\n-{\n-\tstruct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;\n-\tstruct rte_security_docsis_xform *docsis = &conf->docsis;\n-\n-\t/* Downlink: CRC generate -> Cipher encrypt */\n-\tif (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {\n-\n-\t\tif (crypto_sym != NULL &&\n-\t\t    crypto_sym->type ==\tRTE_CRYPTO_SYM_XFORM_CIPHER &&\n-\t\t    crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&\n-\t\t    crypto_sym->cipher.algo ==\n-\t\t\t\t\tRTE_CRYPTO_CIPHER_AES_DOCSISBPI &&\n-\t\t    (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||\n-\t\t     crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&\n-\t\t    crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&\n-\t\t    crypto_sym->next == NULL) {\n-\t\t\treturn 0;\n-\t\t}\n-\t/* Uplink: Cipher decrypt -> CRC verify */\n-\t} else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {\n-\n-\t\tif (crypto_sym != NULL &&\n-\t\t    crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&\n-\t\t    crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&\n-\t\t    crypto_sym->cipher.algo ==\n-\t\t\t\t\tRTE_CRYPTO_CIPHER_AES_DOCSISBPI &&\n-\t\t    (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||\n-\t\t     crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&\n-\t\t    crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&\n-\t\t    crypto_sym->next == NULL) {\n-\t\t\treturn 0;\n-\t\t}\n-\t}\n-\n-\treturn -EINVAL;\n-}\n-\n-/** Set DOCSIS security session auth (CRC) parameters */\n-static int\n-aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,\n-\t\tstruct rte_security_docsis_xform *xform)\n-{\n-\tif (xform == NULL) {\n-\t\tAESNI_MB_LOG(ERR, \"Invalid DOCSIS xform\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\t/* Select CRC generate/verify */\n-\tif (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) {\n-\t\tsess->auth.algo = IMB_AUTH_DOCSIS_CRC32;\n-\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;\n-\t} else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {\n-\t\tsess->auth.algo = IMB_AUTH_DOCSIS_CRC32;\n-\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;\n-\t} else {\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported DOCSIS direction\");\n-\t\treturn -ENOTSUP;\n-\t}\n-\n-\tsess->auth.req_digest_len = RTE_ETHER_CRC_LEN;\n-\tsess->auth.gen_digest_len = RTE_ETHER_CRC_LEN;\n-\n-\treturn 0;\n-}\n-\n-/**\n- * Parse DOCSIS security session configuration and set private session\n- * parameters\n- */\n-int\n-aesni_mb_set_docsis_sec_session_parameters(\n-\t\t__rte_unused struct rte_cryptodev *dev,\n-\t\tstruct rte_security_session_conf *conf,\n-\t\tvoid *sess)\n-{\n-\tstruct rte_security_docsis_xform *docsis_xform;\n-\tstruct rte_crypto_sym_xform *cipher_xform;\n-\tstruct aesni_mb_session *aesni_sess = sess;\n-\tstruct aesni_mb_private *internals = dev->data->dev_private;\n-\tint ret;\n-\n-\tret = check_docsis_sec_session(conf);\n-\tif (ret) {\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported DOCSIS security configuration\");\n-\t\treturn ret;\n-\t}\n-\n-\tswitch (conf->docsis.direction) {\n-\tcase RTE_SECURITY_DOCSIS_UPLINK:\n-\t\taesni_sess->chain_order = IMB_ORDER_CIPHER_HASH;\n-\t\tdocsis_xform = &conf->docsis;\n-\t\tcipher_xform = conf->crypto_xform;\n-\t\tbreak;\n-\tcase RTE_SECURITY_DOCSIS_DOWNLINK:\n-\t\taesni_sess->chain_order = IMB_ORDER_HASH_CIPHER;\n-\t\tcipher_xform = conf->crypto_xform;\n-\t\tdocsis_xform = &conf->docsis;\n-\t\tbreak;\n-\tdefault:\n-\t\treturn -EINVAL;\n-\t}\n-\n-\t/* Default IV length = 0 */\n-\taesni_sess->iv.length = 0;\n-\n-\tret = aesni_mb_set_docsis_sec_session_auth_parameters(aesni_sess,\n-\t\t\tdocsis_xform);\n-\tif (ret != 0) {\n-\t\tAESNI_MB_LOG(ERR, \"Invalid/unsupported DOCSIS parameters\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\tret = aesni_mb_set_session_cipher_parameters(internals->mb_mgr,\n-\t\t\taesni_sess, cipher_xform);\n-\n-\tif (ret != 0) {\n-\t\tAESNI_MB_LOG(ERR, \"Invalid/unsupported cipher parameters\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\treturn 0;\n-}\n-#endif\n-\n-/**\n- * burst enqueue, place crypto operations on ingress queue for processing.\n- *\n- * @param __qp         Queue Pair to process\n- * @param ops          Crypto operations for processing\n- * @param nb_ops       Number of crypto operations for processing\n- *\n- * @return\n- * - Number of crypto operations enqueued\n- */\n-static uint16_t\n-aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,\n-\t\tuint16_t nb_ops)\n-{\n-\tstruct aesni_mb_qp *qp = __qp;\n-\n-\tunsigned int nb_enqueued;\n-\n-\tnb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,\n-\t\t\t(void **)ops, nb_ops, NULL);\n-\n-\tqp->stats.enqueued_count += nb_enqueued;\n-\n-\treturn nb_enqueued;\n-}\n-\n-/** Get multi buffer session */\n-static inline struct aesni_mb_session *\n-get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)\n-{\n-\tstruct aesni_mb_session *sess = NULL;\n-\n-\tif (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {\n-\t\tif (likely(op->sym->session != NULL))\n-\t\t\tsess = (struct aesni_mb_session *)\n-\t\t\t\t\tget_sym_session_private_data(\n-\t\t\t\t\top->sym->session,\n-\t\t\t\t\tcryptodev_driver_id);\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-\t} else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {\n-\t\tif (likely(op->sym->sec_session != NULL))\n-\t\t\tsess = (struct aesni_mb_session *)\n-\t\t\t\t\tget_sec_session_private_data(\n-\t\t\t\t\t\top->sym->sec_session);\n-#endif\n-\t} else {\n-\t\tvoid *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);\n-\t\tvoid *_sess_private_data = NULL;\n-\n-\t\tif (_sess == NULL)\n-\t\t\treturn NULL;\n-\n-\t\tif (rte_mempool_get(qp->sess_mp_priv,\n-\t\t\t\t(void **)&_sess_private_data))\n-\t\t\treturn NULL;\n-\n-\t\tsess = (struct aesni_mb_session *)_sess_private_data;\n-\n-\t\tif (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,\n-\t\t\t\tsess, op->sym->xform) != 0)) {\n-\t\t\trte_mempool_put(qp->sess_mp, _sess);\n-\t\t\trte_mempool_put(qp->sess_mp_priv, _sess_private_data);\n-\t\t\tsess = NULL;\n-\t\t}\n-\t\top->sym->session = (struct rte_cryptodev_sym_session *)_sess;\n-\t\tset_sym_session_private_data(op->sym->session,\n-\t\t\t\tcryptodev_driver_id, _sess_private_data);\n-\t}\n-\n-\tif (unlikely(sess == NULL))\n-\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n-\n-\treturn sess;\n-}\n-\n-static inline uint64_t\n-auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,\n-\t\tuint32_t oop)\n-{\n-\tstruct rte_mbuf *m_src, *m_dst;\n-\tuint8_t *p_src, *p_dst;\n-\tuintptr_t u_src, u_dst;\n-\tuint32_t cipher_end, auth_end;\n-\n-\t/* Only cipher then hash needs special calculation. */\n-\tif (!oop || session->chain_order != CIPHER_HASH)\n-\t\treturn op->sym->auth.data.offset;\n-\n-\tm_src = op->sym->m_src;\n-\tm_dst = op->sym->m_dst;\n-\n-\tp_src = rte_pktmbuf_mtod(m_src, uint8_t *);\n-\tp_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);\n-\tu_src = (uintptr_t)p_src;\n-\tu_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;\n-\n-\t/**\n-\t * Copy the content between cipher offset and auth offset for generating\n-\t * correct digest.\n-\t */\n-\tif (op->sym->cipher.data.offset > op->sym->auth.data.offset)\n-\t\tmemcpy(p_dst + op->sym->auth.data.offset,\n-\t\t\t\tp_src + op->sym->auth.data.offset,\n-\t\t\t\top->sym->cipher.data.offset -\n-\t\t\t\top->sym->auth.data.offset);\n-\n-\t/**\n-\t * Copy the content between (cipher offset + length) and (auth offset +\n-\t * length) for generating correct digest\n-\t */\n-\tcipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;\n-\tauth_end = op->sym->auth.data.offset + op->sym->auth.data.length;\n-\tif (cipher_end < auth_end)\n-\t\tmemcpy(p_dst + cipher_end, p_src + cipher_end,\n-\t\t\t\tauth_end - cipher_end);\n-\n-\t/**\n-\t * Since intel-ipsec-mb only supports positive values,\n-\t * we need to deduct the correct offset between src and dst.\n-\t */\n-\n-\treturn u_src < u_dst ? (u_dst - u_src) :\n-\t\t\t(UINT64_MAX - u_src + u_dst + 1);\n-}\n-\n-static inline void\n-set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,\n-\t\tunion rte_crypto_sym_ofs sofs, void *buf, uint32_t len,\n-\t\tstruct rte_crypto_va_iova_ptr *iv,\n-\t\tstruct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)\n-{\n-\t/* Set crypto operation */\n-\tjob->chain_order = session->chain_order;\n-\n-\t/* Set cipher parameters */\n-\tjob->cipher_direction = session->cipher.direction;\n-\tjob->cipher_mode = session->cipher.mode;\n-\n-\tjob->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;\n-\n-\t/* Set authentication parameters */\n-\tjob->hash_alg = session->auth.algo;\n-\tjob->iv = iv->va;\n-\n-\tswitch (job->hash_alg) {\n-\tcase AES_XCBC:\n-\t\tjob->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;\n-\t\tjob->u.XCBC._k2 = session->auth.xcbc.k2;\n-\t\tjob->u.XCBC._k3 = session->auth.xcbc.k3;\n-\n-\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n-\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n-\t\tbreak;\n-\n-\tcase AES_CCM:\n-\t\tjob->u.CCM.aad = (uint8_t *)aad->va + 18;\n-\t\tjob->u.CCM.aad_len_in_bytes = session->aead.aad_len;\n-\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n-\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n-\t\tjob->iv++;\n-\t\tbreak;\n-\n-\tcase AES_CMAC:\n-\t\tjob->u.CMAC._key_expanded = session->auth.cmac.expkey;\n-\t\tjob->u.CMAC._skey1 = session->auth.cmac.skey1;\n-\t\tjob->u.CMAC._skey2 = session->auth.cmac.skey2;\n-\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n-\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n-\t\tbreak;\n-\n-\tcase AES_GMAC:\n-\t\tif (session->cipher.mode == GCM) {\n-\t\t\tjob->u.GCM.aad = aad->va;\n-\t\t\tjob->u.GCM.aad_len_in_bytes = session->aead.aad_len;\n-\t\t} else {\n-\t\t\t/* For GMAC */\n-\t\t\tjob->u.GCM.aad = buf;\n-\t\t\tjob->u.GCM.aad_len_in_bytes = len;\n-\t\t\tjob->cipher_mode = GCM;\n-\t\t}\n-\t\tjob->aes_enc_key_expanded = &session->cipher.gcm_key;\n-\t\tjob->aes_dec_key_expanded = &session->cipher.gcm_key;\n-\t\tbreak;\n-\n-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM\n-\tcase IMB_AUTH_CHACHA20_POLY1305:\n-\t\tjob->u.CHACHA20_POLY1305.aad = aad->va;\n-\t\tjob->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;\n-\t\tjob->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;\n-\t\tjob->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;\n-\t\tbreak;\n-#endif\n-\tdefault:\n-\t\tjob->u.HMAC._hashed_auth_key_xor_ipad =\n-\t\t\t\tsession->auth.pads.inner;\n-\t\tjob->u.HMAC._hashed_auth_key_xor_opad =\n-\t\t\t\tsession->auth.pads.outer;\n-\n-\t\tif (job->cipher_mode == DES3) {\n-\t\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.exp_3des_keys.ks_ptr;\n-\t\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.exp_3des_keys.ks_ptr;\n-\t\t} else {\n-\t\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n-\t\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n-\t\t}\n-\t}\n-\n-\t/*\n-\t * Multi-buffer library current only support returning a truncated\n-\t * digest length as specified in the relevant IPsec RFCs\n-\t */\n-\n-\t/* Set digest location and length */\n-\tjob->auth_tag_output = digest;\n-\tjob->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;\n-\n-\t/* Set IV parameters */\n-\tjob->iv_len_in_bytes = session->iv.length;\n-\n-\t/* Data Parameters */\n-\tjob->src = buf;\n-\tjob->dst = (uint8_t *)buf + sofs.ofs.cipher.head;\n-\tjob->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;\n-\tjob->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;\n-\tif (job->hash_alg == AES_GMAC && session->cipher.mode != GCM) {\n-\t\tjob->msg_len_to_hash_in_bytes = 0;\n-\t\tjob->msg_len_to_cipher_in_bytes = 0;\n-\t} else {\n-\t\tjob->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -\n-\t\t\tsofs.ofs.auth.tail;\n-\t\tjob->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -\n-\t\t\tsofs.ofs.cipher.tail;\n-\t}\n-\n-\tjob->user_data = udata;\n-}\n-\n-/**\n- * Process a crypto operation and complete a JOB_AES_HMAC job structure for\n- * submission to the multi buffer library for processing.\n- *\n- * @param\tqp\tqueue pair\n- * @param\tjob\tJOB_AES_HMAC structure to fill\n- * @param\tm\tmbuf to process\n- *\n- * @return\n- * - Completed JOB_AES_HMAC structure pointer on success\n- * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible\n- */\n-static inline int\n-set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,\n-\t\tstruct rte_crypto_op *op, uint8_t *digest_idx)\n-{\n-\tstruct rte_mbuf *m_src = op->sym->m_src, *m_dst;\n-\tstruct aesni_mb_session *session;\n-\tuint32_t m_offset, oop;\n-\n-\tsession = get_session(qp, op);\n-\tif (session == NULL) {\n-\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n-\t\treturn -1;\n-\t}\n-\n-\t/* Set crypto operation */\n-\tjob->chain_order = session->chain_order;\n-\n-\t/* Set cipher parameters */\n-\tjob->cipher_direction = session->cipher.direction;\n-\tjob->cipher_mode = session->cipher.mode;\n-\n-\tjob->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;\n-\n-\t/* Set authentication parameters */\n-\tjob->hash_alg = session->auth.algo;\n-\n-\tconst int aead = is_aead_algo(job->hash_alg, job->cipher_mode);\n-\n-\tswitch (job->hash_alg) {\n-\tcase AES_XCBC:\n-\t\tjob->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;\n-\t\tjob->u.XCBC._k2 = session->auth.xcbc.k2;\n-\t\tjob->u.XCBC._k3 = session->auth.xcbc.k3;\n-\n-\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n-\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n-\t\tbreak;\n-\n-\tcase AES_CCM:\n-\t\tjob->u.CCM.aad = op->sym->aead.aad.data + 18;\n-\t\tjob->u.CCM.aad_len_in_bytes = session->aead.aad_len;\n-\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n-\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n-\t\tbreak;\n-\n-\tcase AES_CMAC:\n-\t\tjob->u.CMAC._key_expanded = session->auth.cmac.expkey;\n-\t\tjob->u.CMAC._skey1 = session->auth.cmac.skey1;\n-\t\tjob->u.CMAC._skey2 = session->auth.cmac.skey2;\n-\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n-\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n-\t\tbreak;\n-\n-\tcase AES_GMAC:\n-\t\tif (session->cipher.mode == GCM) {\n-\t\t\tjob->u.GCM.aad = op->sym->aead.aad.data;\n-\t\t\tjob->u.GCM.aad_len_in_bytes = session->aead.aad_len;\n-\t\t} else {\n-\t\t\t/* For GMAC */\n-\t\t\tjob->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,\n-\t\t\t\t\tuint8_t *, op->sym->auth.data.offset);\n-\t\t\tjob->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;\n-\t\t\tjob->cipher_mode = GCM;\n-\t\t}\n-\t\tjob->aes_enc_key_expanded = &session->cipher.gcm_key;\n-\t\tjob->aes_dec_key_expanded = &session->cipher.gcm_key;\n-\t\tbreak;\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\tcase IMB_AUTH_ZUC_EIA3_BITLEN:\n-\t\tjob->u.ZUC_EIA3._key = session->auth.zuc_auth_key;\n-\t\tjob->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n-\t\t\t\t\t\tsession->auth_iv.offset);\n-\t\tbreak;\n-\tcase IMB_AUTH_SNOW3G_UIA2_BITLEN:\n-\t\tjob->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth;\n-\t\tjob->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n-\t\t\t\t\t\tsession->auth_iv.offset);\n-\t\tbreak;\n-\tcase IMB_AUTH_KASUMI_UIA1:\n-\t\tjob->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth;\n-\t\tbreak;\n-#endif\n-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM\n-\tcase IMB_AUTH_CHACHA20_POLY1305:\n-\t\tjob->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;\n-\t\tjob->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;\n-\t\tjob->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;\n-\t\tjob->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;\n-\t\tbreak;\n-#endif\n-\tdefault:\n-\t\tjob->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;\n-\t\tjob->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;\n-\n-\t\tif (job->cipher_mode == DES3) {\n-\t\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.exp_3des_keys.ks_ptr;\n-\t\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.exp_3des_keys.ks_ptr;\n-\t\t} else {\n-\t\t\tjob->aes_enc_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.encode;\n-\t\t\tjob->aes_dec_key_expanded =\n-\t\t\t\tsession->cipher.expanded_aes_keys.decode;\n-\t\t}\n-\t}\n-\n-\tif (aead)\n-\t\tm_offset = op->sym->aead.data.offset;\n-\telse\n-\t\tm_offset = op->sym->cipher.data.offset;\n-\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\tif (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {\n-\t\tjob->aes_enc_key_expanded = session->cipher.zuc_cipher_key;\n-\t\tjob->aes_dec_key_expanded = session->cipher.zuc_cipher_key;\n-\t} else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {\n-\t\tjob->enc_keys = &session->cipher.pKeySched_snow3g_cipher;\n-\t\tm_offset = 0;\n-\t} else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) {\n-\t\tjob->enc_keys = &session->cipher.pKeySched_kasumi_cipher;\n-\t\tm_offset = 0;\n-\t}\n-#endif\n-\n-\tif (!op->sym->m_dst) {\n-\t\t/* in-place operation */\n-\t\tm_dst = m_src;\n-\t\toop = 0;\n-\t} else if (op->sym->m_dst == op->sym->m_src) {\n-\t\t/* in-place operation */\n-\t\tm_dst = m_src;\n-\t\toop = 0;\n-\t} else {\n-\t\t/* out-of-place operation */\n-\t\tm_dst = op->sym->m_dst;\n-\t\toop = 1;\n-\t}\n-\n-\t/* Set digest output location */\n-\tif (job->hash_alg != NULL_HASH &&\n-\t\t\tsession->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {\n-\t\tjob->auth_tag_output = qp->temp_digests[*digest_idx];\n-\t\t*digest_idx = (*digest_idx + 1) % MAX_JOBS;\n-\t} else {\n-\t\tif (aead)\n-\t\t\tjob->auth_tag_output = op->sym->aead.digest.data;\n-\t\telse\n-\t\t\tjob->auth_tag_output = op->sym->auth.digest.data;\n-\n-\t\tif (session->auth.req_digest_len != session->auth.gen_digest_len) {\n-\t\t\tjob->auth_tag_output = qp->temp_digests[*digest_idx];\n-\t\t\t*digest_idx = (*digest_idx + 1) % MAX_JOBS;\n-\t\t}\n-\t}\n-\t/*\n-\t * Multi-buffer library current only support returning a truncated\n-\t * digest length as specified in the relevant IPsec RFCs\n-\t */\n-\n-\t/* Set digest length */\n-\tjob->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;\n-\n-\t/* Set IV parameters */\n-\tjob->iv_len_in_bytes = session->iv.length;\n-\n-\t/* Data Parameters */\n-\tjob->src = rte_pktmbuf_mtod(m_src, uint8_t *);\n-\tjob->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);\n-\n-\tswitch (job->hash_alg) {\n-\tcase AES_CCM:\n-\t\tjob->cipher_start_src_offset_in_bytes =\n-\t\t\t\top->sym->aead.data.offset;\n-\t\tjob->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;\n-\t\tjob->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;\n-\t\tjob->msg_len_to_hash_in_bytes = op->sym->aead.data.length;\n-\n-\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n-\t\t\tsession->iv.offset + 1);\n-\t\tbreak;\n-\n-\tcase AES_GMAC:\n-\t\tif (session->cipher.mode == GCM) {\n-\t\t\tjob->cipher_start_src_offset_in_bytes =\n-\t\t\t\t\top->sym->aead.data.offset;\n-\t\t\tjob->hash_start_src_offset_in_bytes =\n-\t\t\t\t\top->sym->aead.data.offset;\n-\t\t\tjob->msg_len_to_cipher_in_bytes =\n-\t\t\t\t\top->sym->aead.data.length;\n-\t\t\tjob->msg_len_to_hash_in_bytes =\n-\t\t\t\t\top->sym->aead.data.length;\n-\t\t} else {\n-\t\t\tjob->cipher_start_src_offset_in_bytes =\n-\t\t\t\t\top->sym->auth.data.offset;\n-\t\t\tjob->hash_start_src_offset_in_bytes =\n-\t\t\t\t\top->sym->auth.data.offset;\n-\t\t\tjob->msg_len_to_cipher_in_bytes = 0;\n-\t\t\tjob->msg_len_to_hash_in_bytes = 0;\n-\t\t}\n-\n-\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n-\t\t\t\tsession->iv.offset);\n-\t\tbreak;\n-\n-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM\n-\tcase IMB_AUTH_CHACHA20_POLY1305:\n-\t\tjob->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset;\n-\t\tjob->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;\n-\t\tjob->msg_len_to_cipher_in_bytes =\n-\t\t\t\top->sym->aead.data.length;\n-\t\tjob->msg_len_to_hash_in_bytes =\n-\t\t\t\t\top->sym->aead.data.length;\n-\n-\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n-\t\t\t\tsession->iv.offset);\n-\t\tbreak;\n-#endif\n-\tdefault:\n-\t\t/* For SNOW3G, length and offsets are already in bits */\n-\t\tjob->cipher_start_src_offset_in_bytes =\n-\t\t\t\top->sym->cipher.data.offset;\n-\t\tjob->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;\n-\n-\t\tjob->hash_start_src_offset_in_bytes = auth_start_offset(op,\n-\t\t\t\tsession, oop);\n-\t\tjob->msg_len_to_hash_in_bytes = op->sym->auth.data.length;\n-\n-\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n-\t\t\tsession->iv.offset);\n-\t}\n-\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\tif (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)\n-\t\tjob->msg_len_to_cipher_in_bytes >>= 3;\n-\telse if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)\n-\t\tjob->msg_len_to_hash_in_bytes >>= 3;\n-#endif\n-\n-\t/* Set user data to be crypto operation data struct */\n-\tjob->user_data = op;\n-\n-\treturn 0;\n-}\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-/**\n- * Process a crypto operation containing a security op and complete a\n- * JOB_AES_HMAC job structure for submission to the multi buffer library for\n- * processing.\n- */\n-static inline int\n-set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,\n-\t\tstruct rte_crypto_op *op, uint8_t *digest_idx)\n-{\n-\tstruct rte_mbuf *m_src, *m_dst;\n-\tstruct rte_crypto_sym_op *sym;\n-\tstruct aesni_mb_session *session;\n-\n-\tsession = get_session(qp, op);\n-\tif (unlikely(session == NULL)) {\n-\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n-\t\treturn -1;\n-\t}\n-\n-\t/* Only DOCSIS protocol operations supported now */\n-\tif (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||\n-\t\t\tsession->auth.algo != IMB_AUTH_DOCSIS_CRC32) {\n-\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n-\t\treturn -1;\n-\t}\n-\n-\tsym = op->sym;\n-\tm_src = sym->m_src;\n-\n-\tif (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {\n-\t\t/* in-place operation */\n-\t\tm_dst = m_src;\n-\t} else {\n-\t\t/* out-of-place operation not supported */\n-\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n-\t\treturn -ENOTSUP;\n-\t}\n-\n-\t/* Set crypto operation */\n-\tjob->chain_order = session->chain_order;\n-\n-\t/* Set cipher parameters */\n-\tjob->cipher_direction = session->cipher.direction;\n-\tjob->cipher_mode = session->cipher.mode;\n-\n-\tjob->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;\n-\tjob->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;\n-\tjob->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;\n-\n-\t/* Set IV parameters */\n-\tjob->iv_len_in_bytes = session->iv.length;\n-\tjob->iv = (uint8_t *)op + session->iv.offset;\n-\n-\t/* Set authentication parameters */\n-\tjob->hash_alg = session->auth.algo;\n-\n-\t/* Set digest output location */\n-\tjob->auth_tag_output = qp->temp_digests[*digest_idx];\n-\t*digest_idx = (*digest_idx + 1) % MAX_JOBS;\n-\n-\t/* Set digest length */\n-\tjob->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;\n-\n-\t/* Set data parameters */\n-\tjob->src = rte_pktmbuf_mtod(m_src, uint8_t *);\n-\tjob->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,\n-\t\t\t\t\t\tsym->cipher.data.offset);\n-\n-\tjob->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;\n-\tjob->msg_len_to_cipher_in_bytes = sym->cipher.data.length;\n-\n-\tjob->hash_start_src_offset_in_bytes = sym->auth.data.offset;\n-\tjob->msg_len_to_hash_in_bytes = sym->auth.data.length;\n-\n-\tjob->user_data = op;\n-\n-\treturn 0;\n-}\n-\n-static inline void\n-verify_docsis_sec_crc(JOB_AES_HMAC *job, uint8_t *status)\n-{\n-\tuint16_t crc_offset;\n-\tuint8_t *crc;\n-\n-\tif (!job->msg_len_to_hash_in_bytes)\n-\t\treturn;\n-\n-\tcrc_offset = job->hash_start_src_offset_in_bytes +\n-\t\t\tjob->msg_len_to_hash_in_bytes -\n-\t\t\tjob->cipher_start_src_offset_in_bytes;\n-\tcrc = job->dst + crc_offset;\n-\n-\t/* Verify CRC (at the end of the message) */\n-\tif (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0)\n-\t\t*status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n-}\n-#endif\n-\n-static inline void\n-verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)\n-{\n-\t/* Verify digest if required */\n-\tif (memcmp(job->auth_tag_output, digest, len) != 0)\n-\t\t*status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n-}\n-\n-static inline void\n-generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,\n-\t\tstruct aesni_mb_session *sess)\n-{\n-\t/* No extra copy needed */\n-\tif (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))\n-\t\treturn;\n-\n-\t/*\n-\t * This can only happen for HMAC, so only digest\n-\t * for authentication algos is required\n-\t */\n-\tmemcpy(op->sym->auth.digest.data, job->auth_tag_output,\n-\t\t\tsess->auth.req_digest_len);\n-}\n-\n-/**\n- * Process a completed job and return rte_mbuf which job processed\n- *\n- * @param qp\t\tQueue Pair to process\n- * @param job\tJOB_AES_HMAC job to process\n- *\n- * @return\n- * - Returns processed crypto operation.\n- * - Returns NULL on invalid job\n- */\n-static inline struct rte_crypto_op *\n-post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)\n-{\n-\tstruct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;\n-\tstruct aesni_mb_session *sess = NULL;\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-\tuint8_t is_docsis_sec = 0;\n-\n-\tif (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {\n-\t\t/*\n-\t\t * Assuming at this point that if it's a security type op, that\n-\t\t * this is for DOCSIS\n-\t\t */\n-\t\tis_docsis_sec = 1;\n-\t\tsess = get_sec_session_private_data(op->sym->sec_session);\n-\t} else\n-#endif\n-\t{\n-\t\tsess = get_sym_session_private_data(op->sym->session,\n-\t\t\t\t\t\tcryptodev_driver_id);\n-\t}\n-\n-\tif (unlikely(sess == NULL)) {\n-\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n-\t\treturn op;\n-\t}\n-\n-\tif (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {\n-\t\tswitch (job->status) {\n-\t\tcase STS_COMPLETED:\n-\t\t\top->status = RTE_CRYPTO_OP_STATUS_SUCCESS;\n-\n-\t\t\tif (job->hash_alg == NULL_HASH)\n-\t\t\t\tbreak;\n-\n-\t\t\tif (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {\n-\t\t\t\tif (is_aead_algo(job->hash_alg, sess->cipher.mode))\n-\t\t\t\t\tverify_digest(job,\n-\t\t\t\t\t\top->sym->aead.digest.data,\n-\t\t\t\t\t\tsess->auth.req_digest_len,\n-\t\t\t\t\t\t&op->status);\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-\t\t\t\telse if (is_docsis_sec)\n-\t\t\t\t\tverify_docsis_sec_crc(job,\n-\t\t\t\t\t\t&op->status);\n-#endif\n-\t\t\t\telse\n-\t\t\t\t\tverify_digest(job,\n-\t\t\t\t\t\top->sym->auth.digest.data,\n-\t\t\t\t\t\tsess->auth.req_digest_len,\n-\t\t\t\t\t\t&op->status);\n-\t\t\t} else\n-\t\t\t\tgenerate_digest(job, op, sess);\n-\t\t\tbreak;\n-\t\tdefault:\n-\t\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n-\t\t}\n-\t}\n-\n-\t/* Free session if a session-less crypto op */\n-\tif (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {\n-\t\tmemset(sess, 0, sizeof(struct aesni_mb_session));\n-\t\tmemset(op->sym->session, 0,\n-\t\t\trte_cryptodev_sym_get_existing_header_session_size(\n-\t\t\t\top->sym->session));\n-\t\trte_mempool_put(qp->sess_mp_priv, sess);\n-\t\trte_mempool_put(qp->sess_mp, op->sym->session);\n-\t\top->sym->session = NULL;\n-\t}\n-\n-\treturn op;\n-}\n-\n-static inline void\n-post_process_mb_sync_job(JOB_AES_HMAC *job)\n-{\n-\tuint32_t *st;\n-\n-\tst = job->user_data;\n-\tst[0] = (job->status == STS_COMPLETED) ? 0 : EBADMSG;\n-}\n-\n-/**\n- * Process a completed JOB_AES_HMAC job and keep processing jobs until\n- * get_completed_job return NULL\n- *\n- * @param qp\t\tQueue Pair to process\n- * @param job\t\tJOB_AES_HMAC job\n- *\n- * @return\n- * - Number of processed jobs\n- */\n-static unsigned\n-handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,\n-\t\tstruct rte_crypto_op **ops, uint16_t nb_ops)\n-{\n-\tstruct rte_crypto_op *op = NULL;\n-\tunsigned processed_jobs = 0;\n-\n-\twhile (job != NULL) {\n-\t\top = post_process_mb_job(qp, job);\n-\n-\t\tif (op) {\n-\t\t\tops[processed_jobs++] = op;\n-\t\t\tqp->stats.dequeued_count++;\n-\t\t} else {\n-\t\t\tqp->stats.dequeue_err_count++;\n-\t\t\tbreak;\n-\t\t}\n-\t\tif (processed_jobs == nb_ops)\n-\t\t\tbreak;\n-\n-\t\tjob = IMB_GET_COMPLETED_JOB(qp->mb_mgr);\n-\t}\n-\n-\treturn processed_jobs;\n-}\n-\n-static inline uint32_t\n-handle_completed_sync_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr)\n-{\n-\tuint32_t i;\n-\n-\tfor (i = 0; job != NULL; i++, job = IMB_GET_COMPLETED_JOB(mb_mgr))\n-\t\tpost_process_mb_sync_job(job);\n-\n-\treturn i;\n-}\n-\n-static inline uint32_t\n-flush_mb_sync_mgr(MB_MGR *mb_mgr)\n-{\n-\tJOB_AES_HMAC *job;\n-\n-\tjob = IMB_FLUSH_JOB(mb_mgr);\n-\treturn handle_completed_sync_jobs(job, mb_mgr);\n-}\n-\n-static inline uint16_t\n-flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,\n-\t\tuint16_t nb_ops)\n-{\n-\tint processed_ops = 0;\n-\n-\t/* Flush the remaining jobs */\n-\tJOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);\n-\n-\tif (job)\n-\t\tprocessed_ops += handle_completed_jobs(qp, job,\n-\t\t\t\t&ops[processed_ops], nb_ops - processed_ops);\n-\n-\treturn processed_ops;\n-}\n-\n-static inline JOB_AES_HMAC *\n-set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)\n-{\n-\tjob->chain_order = HASH_CIPHER;\n-\tjob->cipher_mode = NULL_CIPHER;\n-\tjob->hash_alg = NULL_HASH;\n-\tjob->cipher_direction = DECRYPT;\n-\n-\t/* Set user data to be crypto operation data struct */\n-\tjob->user_data = op;\n-\n-\treturn job;\n-}\n-\n-static uint16_t\n-aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,\n-\t\tuint16_t nb_ops)\n-{\n-\tstruct aesni_mb_qp *qp = queue_pair;\n-\n-\tstruct rte_crypto_op *op;\n-\tJOB_AES_HMAC *job;\n-\n-\tint retval, processed_jobs = 0;\n-\n-\tif (unlikely(nb_ops == 0))\n-\t\treturn 0;\n-\n-\tuint8_t digest_idx = qp->digest_idx;\n-\tdo {\n-\t\t/* Get next free mb job struct from mb manager */\n-\t\tjob = IMB_GET_NEXT_JOB(qp->mb_mgr);\n-\t\tif (unlikely(job == NULL)) {\n-\t\t\t/* if no free mb job structs we need to flush mb_mgr */\n-\t\t\tprocessed_jobs += flush_mb_mgr(qp,\n-\t\t\t\t\t&ops[processed_jobs],\n-\t\t\t\t\tnb_ops - processed_jobs);\n-\n-\t\t\tif (nb_ops == processed_jobs)\n-\t\t\t\tbreak;\n-\n-\t\t\tjob = IMB_GET_NEXT_JOB(qp->mb_mgr);\n-\t\t}\n-\n-\t\t/*\n-\t\t * Get next operation to process from ingress queue.\n-\t\t * There is no need to return the job to the MB_MGR\n-\t\t * if there are no more operations to process, since the MB_MGR\n-\t\t * can use that pointer again in next get_next calls.\n-\t\t */\n-\t\tretval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);\n-\t\tif (retval < 0)\n-\t\t\tbreak;\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-\t\tif (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)\n-\t\t\tretval = set_sec_mb_job_params(job, qp, op,\n-\t\t\t\t\t\t&digest_idx);\n-\t\telse\n-#endif\n-\t\t\tretval = set_mb_job_params(job, qp, op, &digest_idx);\n-\n-\t\tif (unlikely(retval != 0)) {\n-\t\t\tqp->stats.dequeue_err_count++;\n-\t\t\tset_job_null_op(job, op);\n-\t\t}\n-\n-\t\t/* Submit job to multi-buffer for processing */\n-#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG\n-\t\tjob = IMB_SUBMIT_JOB(qp->mb_mgr);\n-#else\n-\t\tjob = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);\n-#endif\n-\t\t/*\n-\t\t * If submit returns a processed job then handle it,\n-\t\t * before submitting subsequent jobs\n-\t\t */\n-\t\tif (job)\n-\t\t\tprocessed_jobs += handle_completed_jobs(qp, job,\n-\t\t\t\t\t&ops[processed_jobs],\n-\t\t\t\t\tnb_ops - processed_jobs);\n-\n-\t} while (processed_jobs < nb_ops);\n-\n-\tqp->digest_idx = digest_idx;\n-\n-\tif (processed_jobs < 1)\n-\t\tprocessed_jobs += flush_mb_mgr(qp,\n-\t\t\t\t&ops[processed_jobs],\n-\t\t\t\tnb_ops - processed_jobs);\n-\n-\treturn processed_jobs;\n-}\n-\n-static MB_MGR *\n-alloc_init_mb_mgr(enum aesni_mb_vector_mode vector_mode)\n-{\n-\tMB_MGR *mb_mgr = alloc_mb_mgr(0);\n-\tif (mb_mgr == NULL)\n-\t\treturn NULL;\n-\n-\tswitch (vector_mode) {\n-\tcase RTE_AESNI_MB_SSE:\n-\t\tinit_mb_mgr_sse(mb_mgr);\n-\t\tbreak;\n-\tcase RTE_AESNI_MB_AVX:\n-\t\tinit_mb_mgr_avx(mb_mgr);\n-\t\tbreak;\n-\tcase RTE_AESNI_MB_AVX2:\n-\t\tinit_mb_mgr_avx2(mb_mgr);\n-\t\tbreak;\n-\tcase RTE_AESNI_MB_AVX512:\n-\t\tinit_mb_mgr_avx512(mb_mgr);\n-\t\tbreak;\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported vector mode %u\\n\", vector_mode);\n-\t\tfree_mb_mgr(mb_mgr);\n-\t\treturn NULL;\n-\t}\n-\n-\treturn mb_mgr;\n-}\n-\n-static inline void\n-aesni_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)\n-{\n-\tuint32_t i;\n-\n-\tfor (i = 0; i != vec->num; ++i)\n-\t\tvec->status[i] = err;\n-}\n-\n-static inline int\n-check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)\n-{\n-\t/* no multi-seg support with current AESNI-MB PMD */\n-\tif (sgl->num != 1)\n-\t\treturn ENOTSUP;\n-\telse if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)\n-\t\treturn EINVAL;\n-\treturn 0;\n-}\n-\n-static inline JOB_AES_HMAC *\n-submit_sync_job(MB_MGR *mb_mgr)\n-{\n-#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG\n-\treturn IMB_SUBMIT_JOB(mb_mgr);\n-#else\n-\treturn IMB_SUBMIT_JOB_NOCHECK(mb_mgr);\n-#endif\n-}\n-\n-static inline uint32_t\n-generate_sync_dgst(struct rte_crypto_sym_vec *vec,\n-\tconst uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)\n-{\n-\tuint32_t i, k;\n-\n-\tfor (i = 0, k = 0; i != vec->num; i++) {\n-\t\tif (vec->status[i] == 0) {\n-\t\t\tmemcpy(vec->digest[i].va, dgst[i], len);\n-\t\t\tk++;\n-\t\t}\n-\t}\n-\n-\treturn k;\n-}\n-\n-static inline uint32_t\n-verify_sync_dgst(struct rte_crypto_sym_vec *vec,\n-\tconst uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)\n-{\n-\tuint32_t i, k;\n-\n-\tfor (i = 0, k = 0; i != vec->num; i++) {\n-\t\tif (vec->status[i] == 0) {\n-\t\t\tif (memcmp(vec->digest[i].va, dgst[i], len) != 0)\n-\t\t\t\tvec->status[i] = EBADMSG;\n-\t\t\telse\n-\t\t\t\tk++;\n-\t\t}\n-\t}\n-\n-\treturn k;\n-}\n-\n-uint32_t\n-aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,\n-\tstruct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,\n-\tstruct rte_crypto_sym_vec *vec)\n-{\n-\tint32_t ret;\n-\tuint32_t i, j, k, len;\n-\tvoid *buf;\n-\tJOB_AES_HMAC *job;\n-\tMB_MGR *mb_mgr;\n-\tstruct aesni_mb_private *priv;\n-\tstruct aesni_mb_session *s;\n-\tuint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];\n-\n-\ts = get_sym_session_private_data(sess, dev->driver_id);\n-\tif (s == NULL) {\n-\t\taesni_mb_fill_error_code(vec, EINVAL);\n-\t\treturn 0;\n-\t}\n-\n-\t/* get per-thread MB MGR, create one if needed */\n-\tmb_mgr = RTE_PER_LCORE(sync_mb_mgr);\n-\tif (mb_mgr == NULL) {\n-\n-\t\tpriv = dev->data->dev_private;\n-\t\tmb_mgr = alloc_init_mb_mgr(priv->vector_mode);\n-\t\tif (mb_mgr == NULL) {\n-\t\t\taesni_mb_fill_error_code(vec, ENOMEM);\n-\t\t\treturn 0;\n-\t\t}\n-\t\tRTE_PER_LCORE(sync_mb_mgr) = mb_mgr;\n-\t}\n-\n-\tfor (i = 0, j = 0, k = 0; i != vec->num; i++) {\n-\n-\n-\t\tret = check_crypto_sgl(sofs, vec->sgl + i);\n-\t\tif (ret != 0) {\n-\t\t\tvec->status[i] = ret;\n-\t\t\tcontinue;\n-\t\t}\n-\n-\t\tbuf = vec->sgl[i].vec[0].base;\n-\t\tlen = vec->sgl[i].vec[0].len;\n-\n-\t\tjob = IMB_GET_NEXT_JOB(mb_mgr);\n-\t\tif (job == NULL) {\n-\t\t\tk += flush_mb_sync_mgr(mb_mgr);\n-\t\t\tjob = IMB_GET_NEXT_JOB(mb_mgr);\n-\t\t\tRTE_ASSERT(job != NULL);\n-\t\t}\n-\n-\t\t/* Submit job for processing */\n-\t\tset_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i],\n-\t\t\t&vec->aad[i], tmp_dgst[i], &vec->status[i]);\n-\t\tjob = submit_sync_job(mb_mgr);\n-\t\tj++;\n-\n-\t\t/* handle completed jobs */\n-\t\tk += handle_completed_sync_jobs(job, mb_mgr);\n-\t}\n-\n-\t/* flush remaining jobs */\n-\twhile (k != j)\n-\t\tk += flush_mb_sync_mgr(mb_mgr);\n-\n-\t/* finish processing for successful jobs: check/update digest */\n-\tif (k != 0) {\n-\t\tif (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)\n-\t\t\tk = verify_sync_dgst(vec,\n-\t\t\t\t(const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,\n-\t\t\t\ts->auth.req_digest_len);\n-\t\telse\n-\t\t\tk = generate_sync_dgst(vec,\n-\t\t\t\t(const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,\n-\t\t\t\ts->auth.req_digest_len);\n-\t}\n-\n-\treturn k;\n-}\n-\n-static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);\n-\n-static uint64_t\n-vec_mode_to_flags(enum aesni_mb_vector_mode mode)\n-{\n-\tswitch (mode) {\n-\tcase RTE_AESNI_MB_SSE:\n-\t\treturn RTE_CRYPTODEV_FF_CPU_SSE;\n-\tcase RTE_AESNI_MB_AVX:\n-\t\treturn RTE_CRYPTODEV_FF_CPU_AVX;\n-\tcase RTE_AESNI_MB_AVX2:\n-\t\treturn RTE_CRYPTODEV_FF_CPU_AVX2;\n-\tcase RTE_AESNI_MB_AVX512:\n-\t\treturn RTE_CRYPTODEV_FF_CPU_AVX512;\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported vector mode %u\\n\", mode);\n-\t\treturn 0;\n-\t}\n-}\n-\n-static int\n-cryptodev_aesni_mb_create(const char *name,\n-\t\t\tstruct rte_vdev_device *vdev,\n-\t\t\tstruct rte_cryptodev_pmd_init_params *init_params)\n-{\n-\tstruct rte_cryptodev *dev;\n-\tstruct aesni_mb_private *internals;\n-\tenum aesni_mb_vector_mode vector_mode;\n-\tMB_MGR *mb_mgr;\n-\n-\tdev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);\n-\tif (dev == NULL) {\n-\t\tAESNI_MB_LOG(ERR, \"failed to create cryptodev vdev\");\n-\t\treturn -ENODEV;\n-\t}\n-\n-\t/* Check CPU for supported vector instruction set */\n-\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))\n-\t\tvector_mode = RTE_AESNI_MB_AVX512;\n-\telse if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))\n-\t\tvector_mode = RTE_AESNI_MB_AVX2;\n-\telse if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))\n-\t\tvector_mode = RTE_AESNI_MB_AVX;\n-\telse\n-\t\tvector_mode = RTE_AESNI_MB_SSE;\n-\n-\tdev->driver_id = cryptodev_driver_id;\n-\tdev->dev_ops = rte_aesni_mb_pmd_ops;\n-\n-\t/* register rx/tx burst functions for data path */\n-\tdev->dequeue_burst = aesni_mb_pmd_dequeue_burst;\n-\tdev->enqueue_burst = aesni_mb_pmd_enqueue_burst;\n-\n-\tdev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |\n-\t\t\tRTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |\n-\t\t\tRTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |\n-\t\t\tRTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |\n-\t\t\tRTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |\n-\t\t\tRTE_CRYPTODEV_FF_SYM_SESSIONLESS;\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-\tstruct rte_security_ctx *security_instance;\n-\tsecurity_instance = rte_malloc(\"aesni_mb_sec\",\n-\t\t\t\tsizeof(struct rte_security_ctx),\n-\t\t\t\tRTE_CACHE_LINE_SIZE);\n-\tif (security_instance == NULL) {\n-\t\tAESNI_MB_LOG(ERR, \"rte_security_ctx memory alloc failed\");\n-\t\trte_cryptodev_pmd_destroy(dev);\n-\t\treturn -ENOMEM;\n-\t}\n-\n-\tsecurity_instance->device = (void *)dev;\n-\tsecurity_instance->ops = rte_aesni_mb_pmd_sec_ops;\n-\tsecurity_instance->sess_cnt = 0;\n-\tdev->security_ctx = security_instance;\n-\tdev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;\n-#endif\n-\n-\t/* Check CPU for support for AES instruction set */\n-\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))\n-\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;\n-\telse\n-\t\tAESNI_MB_LOG(WARNING, \"AES instructions not supported by CPU\");\n-\n-\tdev->feature_flags |= vec_mode_to_flags(vector_mode);\n-\n-\tmb_mgr = alloc_init_mb_mgr(vector_mode);\n-\tif (mb_mgr == NULL) {\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-\t\trte_free(dev->security_ctx);\n-\t\tdev->security_ctx = NULL;\n-#endif\n-\t\trte_cryptodev_pmd_destroy(dev);\n-\t\treturn -ENOMEM;\n-\t}\n-\n-\t/* Set vector instructions mode supported */\n-\tinternals = dev->data->dev_private;\n-\n-\tinternals->vector_mode = vector_mode;\n-\tinternals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;\n-\tinternals->mb_mgr = mb_mgr;\n-\n-\tAESNI_MB_LOG(INFO, \"IPSec Multi-buffer library version used: %s\\n\",\n-\t\t\timb_get_version_str());\n-\treturn 0;\n-}\n-\n-static int\n-cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)\n-{\n-\tstruct rte_cryptodev_pmd_init_params init_params = {\n-\t\t\"\",\n-\t\tsizeof(struct aesni_mb_private),\n-\t\trte_socket_id(),\n-\t\tRTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS\n-\t};\n-\tconst char *name, *args;\n-\tint retval;\n-\n-\tname = rte_vdev_device_name(vdev);\n-\tif (name == NULL)\n-\t\treturn -EINVAL;\n-\n-\targs = rte_vdev_device_args(vdev);\n-\n-\tretval = rte_cryptodev_pmd_parse_input_args(&init_params, args);\n-\tif (retval) {\n-\t\tAESNI_MB_LOG(ERR, \"Failed to parse initialisation arguments[%s]\",\n-\t\t\t\targs);\n-\t\treturn -EINVAL;\n-\t}\n-\n-\treturn cryptodev_aesni_mb_create(name, vdev, &init_params);\n-}\n-\n-static int\n-cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)\n-{\n-\tstruct rte_cryptodev *cryptodev;\n-\tstruct aesni_mb_private *internals;\n-\tconst char *name;\n-\n-\tname = rte_vdev_device_name(vdev);\n-\tif (name == NULL)\n-\t\treturn -EINVAL;\n-\n-\tcryptodev = rte_cryptodev_pmd_get_named_dev(name);\n-\tif (cryptodev == NULL)\n-\t\treturn -ENODEV;\n-\n-\tinternals = cryptodev->data->dev_private;\n-\n-\tfree_mb_mgr(internals->mb_mgr);\n-\tif (RTE_PER_LCORE(sync_mb_mgr)) {\n-\t\tfree_mb_mgr(RTE_PER_LCORE(sync_mb_mgr));\n-\t\tRTE_PER_LCORE(sync_mb_mgr) = NULL;\n-\t}\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-\trte_free(cryptodev->security_ctx);\n-\tcryptodev->security_ctx = NULL;\n-#endif\n-\n-\treturn rte_cryptodev_pmd_destroy(cryptodev);\n-}\n-\n-static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {\n-\t.probe = cryptodev_aesni_mb_probe,\n-\t.remove = cryptodev_aesni_mb_remove\n-};\n-\n-static struct cryptodev_driver aesni_mb_crypto_drv;\n-\n-RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);\n-RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);\n-RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,\n-\t\"max_nb_queue_pairs=<int> \"\n-\t\"socket_id=<int>\");\n-RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,\n-\t\tcryptodev_aesni_mb_pmd_drv.driver,\n-\t\tcryptodev_driver_id);\n-RTE_LOG_REGISTER_DEFAULT(aesni_mb_logtype_driver, NOTICE);\ndiff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c\ndeleted file mode 100644\nindex 48a8f91868..0000000000\n--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c\n+++ /dev/null\n@@ -1,1126 +0,0 @@\n-/* SPDX-License-Identifier: BSD-3-Clause\n- * Copyright(c) 2015-2017 Intel Corporation\n- */\n-\n-#include <string.h>\n-\n-#include <rte_string_fns.h>\n-#include <rte_common.h>\n-#include <rte_malloc.h>\n-#include <rte_ether.h>\n-#include <cryptodev_pmd.h>\n-\n-#include \"aesni_mb_pmd_private.h\"\n-\n-\n-static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {\n-\t{\t/* MD5 HMAC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_MD5_HMAC,\n-\t\t\t\t.block_size = 64,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 64,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA1 HMAC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA1_HMAC,\n-\t\t\t\t.block_size = 64,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 65535,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 20,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA1 */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA1,\n-\t\t\t\t.block_size = 64,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 0,\n-\t\t\t\t\t.max = 0,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 20,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA224 HMAC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,\n-\t\t\t\t.block_size = 64,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 65535,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 28,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA224 */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA224,\n-\t\t\t\t.block_size = 64,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 0,\n-\t\t\t\t\t.max = 0,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 28,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA256 HMAC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA256_HMAC,\n-\t\t\t\t.block_size = 64,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 65535,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA256 */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA256,\n-\t\t\t\t.block_size = 64,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 0,\n-\t\t\t\t\t.max = 0,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA384 HMAC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA384_HMAC,\n-\t\t\t\t.block_size = 128,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 65535,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 48,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA384 */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA384,\n-\t\t\t\t.block_size = 128,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 0,\n-\t\t\t\t\t.max = 0,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 48,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA512 HMAC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA512_HMAC,\n-\t\t\t\t.block_size = 128,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 65535,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 64,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SHA512  */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA512,\n-\t\t\t\t.block_size = 128,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 0,\n-\t\t\t\t\t.max = 0,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 64,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* AES XCBC HMAC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 12,\n-\t\t\t\t\t.max = 12,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* AES CBC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_CBC,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 8\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* AES CTR */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_CTR,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 8\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 12,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 4\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* AES DOCSIS BPI */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 16\n-#else\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-#endif\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* DES CBC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_DES_CBC,\n-\t\t\t\t.block_size = 8,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 8,\n-\t\t\t\t\t.max = 8,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 8,\n-\t\t\t\t\t.max = 8,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/*  3DES CBC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_3DES_CBC,\n-\t\t\t\t.block_size = 8,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 8,\n-\t\t\t\t\t.max = 24,\n-\t\t\t\t\t.increment = 8\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 8,\n-\t\t\t\t\t.max = 8,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* DES DOCSIS BPI */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,\n-\t\t\t\t.block_size = 8,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 8,\n-\t\t\t\t\t.max = 8,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 8,\n-\t\t\t\t\t.max = 8,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* AES CCM */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n-\t\t\t{.aead = {\n-\t\t\t\t.algo = RTE_CRYPTO_AEAD_AES_CCM,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-#if IMB_VERSION(0, 54, 2) <= IMB_VERSION_NUM\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 16\n-#else\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-#endif\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 4,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 2\n-\t\t\t\t},\n-\t\t\t\t.aad_size = {\n-\t\t\t\t\t.min = 0,\n-\t\t\t\t\t.max = 46,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 7,\n-\t\t\t\t\t.max = 13,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* AES CMAC */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_AES_CMAC,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* AES GCM */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n-\t\t\t{.aead = {\n-\t\t\t\t.algo = RTE_CRYPTO_AEAD_AES_GCM,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 8\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.aad_size = {\n-\t\t\t\t\t.min = 0,\n-\t\t\t\t\t.max = 65535,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 12,\n-\t\t\t\t\t.max = 12,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* AES GMAC (AUTH) */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_AES_GMAC,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 8\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 1,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 12,\n-\t\t\t\t\t.max = 12,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-#if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM\n-\t{\t/* AES ECB */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_ECB,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 8\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-#endif\n-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM\n-\t{\t/* ZUC (EIA3) */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_ZUC_EIA3,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 4,\n-\t\t\t\t\t.max = 4,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* ZUC (EEA3) */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SNOW 3G (UIA2) */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 4,\n-\t\t\t\t\t.max = 4,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* SNOW 3G (UEA2) */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* KASUMI (F9) */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n-\t\t\t{.auth = {\n-\t\t\t\t.algo = RTE_CRYPTO_AUTH_KASUMI_F9,\n-\t\t\t\t.block_size = 8,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 4,\n-\t\t\t\t\t.max = 4,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = { 0 }\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\t{\t/* KASUMI (F8) */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_KASUMI_F8,\n-\t\t\t\t.block_size = 8,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 8,\n-\t\t\t\t\t.max = 8,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-#endif\n-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM\n-\t{\t/* CHACHA20-POLY1305 */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n-\t\t\t{.aead = {\n-\t\t\t\t.algo = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,\n-\t\t\t\t.block_size = 64,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 32,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.digest_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t\t.aad_size = {\n-\t\t\t\t\t.min = 0,\n-\t\t\t\t\t.max = 240,\n-\t\t\t\t\t.increment = 1\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 12,\n-\t\t\t\t\t.max = 12,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t},\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-#endif\n-\tRTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()\n-};\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-static const struct rte_cryptodev_capabilities\n-\t\t\t\t\taesni_mb_pmd_security_crypto_cap[] = {\n-\t{\t/* AES DOCSIS BPI */\n-\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n-\t\t{.sym = {\n-\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n-\t\t\t{.cipher = {\n-\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,\n-\t\t\t\t.block_size = 16,\n-\t\t\t\t.key_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 32,\n-\t\t\t\t\t.increment = 16\n-\t\t\t\t},\n-\t\t\t\t.iv_size = {\n-\t\t\t\t\t.min = 16,\n-\t\t\t\t\t.max = 16,\n-\t\t\t\t\t.increment = 0\n-\t\t\t\t}\n-\t\t\t}, }\n-\t\t}, }\n-\t},\n-\n-\tRTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()\n-};\n-\n-static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {\n-\t{\t/* DOCSIS Uplink */\n-\t\t.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,\n-\t\t.protocol = RTE_SECURITY_PROTOCOL_DOCSIS,\n-\t\t.docsis = {\n-\t\t\t.direction = RTE_SECURITY_DOCSIS_UPLINK\n-\t\t},\n-\t\t.crypto_capabilities = aesni_mb_pmd_security_crypto_cap\n-\t},\n-\t{\t/* DOCSIS Downlink */\n-\t\t.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,\n-\t\t.protocol = RTE_SECURITY_PROTOCOL_DOCSIS,\n-\t\t.docsis = {\n-\t\t\t.direction = RTE_SECURITY_DOCSIS_DOWNLINK\n-\t\t},\n-\t\t.crypto_capabilities = aesni_mb_pmd_security_crypto_cap\n-\t},\n-\t{\n-\t\t.action = RTE_SECURITY_ACTION_TYPE_NONE\n-\t}\n-};\n-#endif\n-\n-/** Configure device */\n-static int\n-aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,\n-\t\t__rte_unused struct rte_cryptodev_config *config)\n-{\n-\treturn 0;\n-}\n-\n-/** Start device */\n-static int\n-aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)\n-{\n-\treturn 0;\n-}\n-\n-/** Stop device */\n-static void\n-aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)\n-{\n-}\n-\n-/** Close device */\n-static int\n-aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)\n-{\n-\treturn 0;\n-}\n-\n-\n-/** Get device statistics */\n-static void\n-aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,\n-\t\tstruct rte_cryptodev_stats *stats)\n-{\n-\tint qp_id;\n-\n-\tfor (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {\n-\t\tstruct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];\n-\n-\t\tstats->enqueued_count += qp->stats.enqueued_count;\n-\t\tstats->dequeued_count += qp->stats.dequeued_count;\n-\n-\t\tstats->enqueue_err_count += qp->stats.enqueue_err_count;\n-\t\tstats->dequeue_err_count += qp->stats.dequeue_err_count;\n-\t}\n-}\n-\n-/** Reset device statistics */\n-static void\n-aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)\n-{\n-\tint qp_id;\n-\n-\tfor (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {\n-\t\tstruct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];\n-\n-\t\tmemset(&qp->stats, 0, sizeof(qp->stats));\n-\t}\n-}\n-\n-\n-/** Get device info */\n-static void\n-aesni_mb_pmd_info_get(struct rte_cryptodev *dev,\n-\t\tstruct rte_cryptodev_info *dev_info)\n-{\n-\tstruct aesni_mb_private *internals = dev->data->dev_private;\n-\n-\tif (dev_info != NULL) {\n-\t\tdev_info->driver_id = dev->driver_id;\n-\t\tdev_info->feature_flags = dev->feature_flags;\n-\t\tdev_info->capabilities = aesni_mb_pmd_capabilities;\n-\t\tdev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;\n-\t\t/* No limit of number of sessions */\n-\t\tdev_info->sym.max_nb_sessions = 0;\n-\t}\n-}\n-\n-/** Release queue pair */\n-static int\n-aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)\n-{\n-\tstruct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];\n-\tstruct rte_ring *r = NULL;\n-\n-\tif (qp != NULL) {\n-\t\tr = rte_ring_lookup(qp->name);\n-\t\tif (r)\n-\t\t\trte_ring_free(r);\n-\t\tif (qp->mb_mgr)\n-\t\t\tfree_mb_mgr(qp->mb_mgr);\n-\t\trte_free(qp);\n-\t\tdev->data->queue_pairs[qp_id] = NULL;\n-\t}\n-\treturn 0;\n-}\n-\n-/** set a unique name for the queue pair based on it's name, dev_id and qp_id */\n-static int\n-aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,\n-\t\tstruct aesni_mb_qp *qp)\n-{\n-\tunsigned n = snprintf(qp->name, sizeof(qp->name),\n-\t\t\t\"aesni_mb_pmd_%u_qp_%u\",\n-\t\t\tdev->data->dev_id, qp->id);\n-\n-\tif (n >= sizeof(qp->name))\n-\t\treturn -1;\n-\n-\treturn 0;\n-}\n-\n-/** Create a ring to place processed operations on */\n-static struct rte_ring *\n-aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,\n-\t\tunsigned int ring_size, int socket_id)\n-{\n-\tstruct rte_ring *r;\n-\tchar ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];\n-\n-\tunsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));\n-\n-\tif (n >= sizeof(ring_name))\n-\t\treturn NULL;\n-\n-\tr = rte_ring_lookup(ring_name);\n-\tif (r) {\n-\t\tif (rte_ring_get_size(r) >= ring_size) {\n-\t\t\tAESNI_MB_LOG(INFO, \"Reusing existing ring %s for processed ops\",\n-\t\t\tring_name);\n-\t\t\treturn r;\n-\t\t}\n-\n-\t\tAESNI_MB_LOG(ERR, \"Unable to reuse existing ring %s for processed ops\",\n-\t\t\tring_name);\n-\t\treturn NULL;\n-\t}\n-\n-\treturn rte_ring_create(ring_name, ring_size, socket_id,\n-\t\t\tRING_F_SP_ENQ | RING_F_SC_DEQ);\n-}\n-\n-/** Setup a queue pair */\n-static int\n-aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,\n-\t\tconst struct rte_cryptodev_qp_conf *qp_conf,\n-\t\tint socket_id)\n-{\n-\tstruct aesni_mb_qp *qp = NULL;\n-\tstruct aesni_mb_private *internals = dev->data->dev_private;\n-\tint ret = -1;\n-\n-\t/* Free memory prior to re-allocation if needed. */\n-\tif (dev->data->queue_pairs[qp_id] != NULL)\n-\t\taesni_mb_pmd_qp_release(dev, qp_id);\n-\n-\t/* Allocate the queue pair data structure. */\n-\tqp = rte_zmalloc_socket(\"AES-NI PMD Queue Pair\", sizeof(*qp),\n-\t\t\t\t\tRTE_CACHE_LINE_SIZE, socket_id);\n-\tif (qp == NULL)\n-\t\treturn -ENOMEM;\n-\n-\tqp->id = qp_id;\n-\tdev->data->queue_pairs[qp_id] = qp;\n-\n-\tif (aesni_mb_pmd_qp_set_unique_name(dev, qp))\n-\t\tgoto qp_setup_cleanup;\n-\n-\n-\tqp->mb_mgr = alloc_mb_mgr(0);\n-\tif (qp->mb_mgr == NULL) {\n-\t\tret = -ENOMEM;\n-\t\tgoto qp_setup_cleanup;\n-\t}\n-\n-\tswitch (internals->vector_mode) {\n-\tcase RTE_AESNI_MB_SSE:\n-\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;\n-\t\tinit_mb_mgr_sse(qp->mb_mgr);\n-\t\tbreak;\n-\tcase RTE_AESNI_MB_AVX:\n-\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;\n-\t\tinit_mb_mgr_avx(qp->mb_mgr);\n-\t\tbreak;\n-\tcase RTE_AESNI_MB_AVX2:\n-\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;\n-\t\tinit_mb_mgr_avx2(qp->mb_mgr);\n-\t\tbreak;\n-\tcase RTE_AESNI_MB_AVX512:\n-\t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;\n-\t\tinit_mb_mgr_avx512(qp->mb_mgr);\n-\t\tbreak;\n-\tdefault:\n-\t\tAESNI_MB_LOG(ERR, \"Unsupported vector mode %u\\n\",\n-\t\t\t\tinternals->vector_mode);\n-\t\tgoto qp_setup_cleanup;\n-\t}\n-\n-\tqp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,\n-\t\t\tqp_conf->nb_descriptors, socket_id);\n-\tif (qp->ingress_queue == NULL) {\n-\t\tret = -1;\n-\t\tgoto qp_setup_cleanup;\n-\t}\n-\n-\tqp->sess_mp = qp_conf->mp_session;\n-\tqp->sess_mp_priv = qp_conf->mp_session_private;\n-\n-\tmemset(&qp->stats, 0, sizeof(qp->stats));\n-\n-\tchar mp_name[RTE_MEMPOOL_NAMESIZE];\n-\n-\tsnprintf(mp_name, RTE_MEMPOOL_NAMESIZE,\n-\t\t\t\t\"digest_mp_%u_%u\", dev->data->dev_id, qp_id);\n-\treturn 0;\n-\n-qp_setup_cleanup:\n-\tif (qp) {\n-\t\tif (qp->mb_mgr)\n-\t\t\tfree_mb_mgr(qp->mb_mgr);\n-\t\trte_free(qp);\n-\t}\n-\n-\treturn ret;\n-}\n-\n-/** Returns the size of the aesni multi-buffer session structure */\n-static unsigned\n-aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)\n-{\n-\treturn sizeof(struct aesni_mb_session);\n-}\n-\n-/** Configure a aesni multi-buffer session from a crypto xform chain */\n-static int\n-aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,\n-\t\tstruct rte_crypto_sym_xform *xform,\n-\t\tstruct rte_cryptodev_sym_session *sess,\n-\t\tstruct rte_mempool *mempool)\n-{\n-\tvoid *sess_private_data;\n-\tstruct aesni_mb_private *internals = dev->data->dev_private;\n-\tint ret;\n-\n-\tif (unlikely(sess == NULL)) {\n-\t\tAESNI_MB_LOG(ERR, \"invalid session struct\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\tif (rte_mempool_get(mempool, &sess_private_data)) {\n-\t\tAESNI_MB_LOG(ERR,\n-\t\t\t\t\"Couldn't get object from session mempool\");\n-\t\treturn -ENOMEM;\n-\t}\n-\n-\tret = aesni_mb_set_session_parameters(internals->mb_mgr,\n-\t\t\tsess_private_data, xform);\n-\tif (ret != 0) {\n-\t\tAESNI_MB_LOG(ERR, \"failed configure session parameters\");\n-\n-\t\t/* Return session to mempool */\n-\t\trte_mempool_put(mempool, sess_private_data);\n-\t\treturn ret;\n-\t}\n-\n-\tset_sym_session_private_data(sess, dev->driver_id,\n-\t\t\tsess_private_data);\n-\n-\treturn 0;\n-}\n-\n-/** Clear the memory of session so it doesn't leave key material behind */\n-static void\n-aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,\n-\t\tstruct rte_cryptodev_sym_session *sess)\n-{\n-\tuint8_t index = dev->driver_id;\n-\tvoid *sess_priv = get_sym_session_private_data(sess, index);\n-\n-\t/* Zero out the whole structure */\n-\tif (sess_priv) {\n-\t\tmemset(sess_priv, 0, sizeof(struct aesni_mb_session));\n-\t\tstruct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);\n-\t\tset_sym_session_private_data(sess, index, NULL);\n-\t\trte_mempool_put(sess_mp, sess_priv);\n-\t}\n-}\n-\n-struct rte_cryptodev_ops aesni_mb_pmd_ops = {\n-\t\t.dev_configure\t\t= aesni_mb_pmd_config,\n-\t\t.dev_start\t\t= aesni_mb_pmd_start,\n-\t\t.dev_stop\t\t= aesni_mb_pmd_stop,\n-\t\t.dev_close\t\t= aesni_mb_pmd_close,\n-\n-\t\t.stats_get\t\t= aesni_mb_pmd_stats_get,\n-\t\t.stats_reset\t\t= aesni_mb_pmd_stats_reset,\n-\n-\t\t.dev_infos_get\t\t= aesni_mb_pmd_info_get,\n-\n-\t\t.queue_pair_setup\t= aesni_mb_pmd_qp_setup,\n-\t\t.queue_pair_release\t= aesni_mb_pmd_qp_release,\n-\n-\t\t.sym_cpu_process\t= aesni_mb_cpu_crypto_process_bulk,\n-\n-\t\t.sym_session_get_size\t= aesni_mb_pmd_sym_session_get_size,\n-\t\t.sym_session_configure\t= aesni_mb_pmd_sym_session_configure,\n-\t\t.sym_session_clear\t= aesni_mb_pmd_sym_session_clear\n-};\n-\n-struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;\n-\n-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n-/**\n- * Configure a aesni multi-buffer session from a security session\n- * configuration\n- */\n-static int\n-aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,\n-\t\tstruct rte_security_session *sess,\n-\t\tstruct rte_mempool *mempool)\n-{\n-\tvoid *sess_private_data;\n-\tstruct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;\n-\tint ret;\n-\n-\tif (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||\n-\t\t\tconf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {\n-\t\tAESNI_MB_LOG(ERR, \"Invalid security protocol\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\tif (rte_mempool_get(mempool, &sess_private_data)) {\n-\t\tAESNI_MB_LOG(ERR, \"Couldn't get object from session mempool\");\n-\t\treturn -ENOMEM;\n-\t}\n-\n-\tret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,\n-\t\t\tsess_private_data);\n-\n-\tif (ret != 0) {\n-\t\tAESNI_MB_LOG(ERR, \"Failed to configure session parameters\");\n-\n-\t\t/* Return session to mempool */\n-\t\trte_mempool_put(mempool, sess_private_data);\n-\t\treturn ret;\n-\t}\n-\n-\tset_sec_session_private_data(sess, sess_private_data);\n-\n-\treturn ret;\n-}\n-\n-/** Clear the memory of session so it doesn't leave key material behind */\n-static int\n-aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,\n-\t\tstruct rte_security_session *sess)\n-{\n-\tvoid *sess_priv = get_sec_session_private_data(sess);\n-\n-\tif (sess_priv) {\n-\t\tstruct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);\n-\t\tmemset(sess_priv, 0, sizeof(struct aesni_mb_session));\n-\t\tset_sec_session_private_data(sess, NULL);\n-\t\trte_mempool_put(sess_mp, sess_priv);\n-\t}\n-\treturn 0;\n-}\n-\n-/** Get security capabilities for aesni multi-buffer */\n-static const struct rte_security_capability *\n-aesni_mb_pmd_sec_capa_get(void *device __rte_unused)\n-{\n-\treturn aesni_mb_pmd_security_cap;\n-}\n-\n-static struct rte_security_ops aesni_mb_pmd_sec_ops = {\n-\t\t.session_create = aesni_mb_pmd_sec_sess_create,\n-\t\t.session_update = NULL,\n-\t\t.session_stats_get = NULL,\n-\t\t.session_destroy = aesni_mb_pmd_sec_sess_destroy,\n-\t\t.set_pkt_metadata = NULL,\n-\t\t.capabilities_get = aesni_mb_pmd_sec_capa_get\n-};\n-\n-struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;\n-#endif\ndiff --git a/drivers/crypto/aesni_mb/version.map b/drivers/crypto/aesni_mb/version.map\ndeleted file mode 100644\nindex c2e0723b4c..0000000000\n--- a/drivers/crypto/aesni_mb/version.map\n+++ /dev/null\n@@ -1,3 +0,0 @@\n-DPDK_22 {\n-\tlocal: *;\n-};\ndiff --git a/drivers/crypto/ipsec_mb/meson.build b/drivers/crypto/ipsec_mb/meson.build\nindex 3d48da60ed..bac5d85e26 100644\n--- a/drivers/crypto/ipsec_mb/meson.build\n+++ b/drivers/crypto/ipsec_mb/meson.build\n@@ -23,5 +23,6 @@ endif\n \n sources = files('rte_ipsec_mb_pmd.c',\n \t\t'rte_ipsec_mb_pmd_ops.c',\n+\t\t'pmd_aesni_mb.c'\n \t\t)\n deps += ['bus_vdev', 'net', 'security']\ndiff --git a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c\nnew file mode 100644\nindex 0000000000..3c377ab753\n--- /dev/null\n+++ b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c\n@@ -0,0 +1,2977 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(c) 2015-2021 Intel Corporation\n+ */\n+\n+#include <intel-ipsec-mb.h>\n+\n+#if defined(RTE_LIB_SECURITY)\n+#define AESNI_MB_DOCSIS_SEC_ENABLED 1\n+#include <rte_security.h>\n+#include <rte_security_driver.h>\n+#include <rte_ether.h>\n+#endif\n+\n+#include \"rte_ipsec_mb_pmd_private.h\"\n+\n+#define AES_CCM_DIGEST_MIN_LEN 4\n+#define AES_CCM_DIGEST_MAX_LEN 16\n+#define HMAC_MAX_BLOCK_SIZE 128\n+#define HMAC_IPAD_VALUE\t\t\t(0x36)\n+#define HMAC_OPAD_VALUE\t\t\t(0x5C)\n+\n+static const struct rte_cryptodev_capabilities aesni_mb_capabilities[] = {\n+\t{\t/* MD5 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_MD5_HMAC,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 64,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA1 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA1_HMAC,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 20,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA1 */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA1,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 0,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 20,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA224 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 28,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA224 */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA224,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 0,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 28,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA256 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA256_HMAC,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA256 */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA256,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 0,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA384 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA384_HMAC,\n+\t\t\t\t.block_size = 128,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 48,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA384 */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA384,\n+\t\t\t\t.block_size = 128,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 0,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 48,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA512 HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA512_HMAC,\n+\t\t\t\t.block_size = 128,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 64,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SHA512  */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SHA512,\n+\t\t\t\t.block_size = 128,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 0,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 64,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES XCBC HMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 12,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES CBC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_CBC,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES CTR */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_CTR,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 4\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES DOCSIS BPI */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 16\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* DES CBC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_DES_CBC,\n+\t\t\t\t.block_size = 8,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/*  3DES CBC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_3DES_CBC,\n+\t\t\t\t.block_size = 8,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 24,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* DES DOCSIS BPI */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,\n+\t\t\t\t.block_size = 8,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES CCM */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n+\t\t\t{.aead = {\n+\t\t\t\t.algo = RTE_CRYPTO_AEAD_AES_CCM,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 16\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 4,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 2\n+\t\t\t\t},\n+\t\t\t\t.aad_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 46,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 7,\n+\t\t\t\t\t.max = 13,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES CMAC */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_AES_CMAC,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES GCM */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n+\t\t\t{.aead = {\n+\t\t\t\t.algo = RTE_CRYPTO_AEAD_AES_GCM,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.aad_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 65535,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 12,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES GMAC (AUTH) */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_AES_GMAC,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 1,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 12,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* AES ECB */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_ECB,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 8\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* ZUC (EIA3) */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_ZUC_EIA3,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 4,\n+\t\t\t\t\t.max = 4,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* ZUC (EEA3) */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SNOW 3G (UIA2) */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 4,\n+\t\t\t\t\t.max = 4,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* SNOW 3G (UEA2) */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* KASUMI (F9) */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,\n+\t\t\t{.auth = {\n+\t\t\t\t.algo = RTE_CRYPTO_AUTH_KASUMI_F9,\n+\t\t\t\t.block_size = 8,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 4,\n+\t\t\t\t\t.max = 4,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = { 0 }\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* KASUMI (F8) */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_KASUMI_F8,\n+\t\t\t\t.block_size = 8,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 8,\n+\t\t\t\t\t.max = 8,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\t{\t/* CHACHA20-POLY1305 */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n+\t\t\t{.aead = {\n+\t\t\t\t.algo = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,\n+\t\t\t\t.block_size = 64,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 32,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.aad_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 240,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 12,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\tRTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()\n+};\n+\n+uint8_t pmd_driver_id_aesni_mb;\n+\n+struct aesni_mb_qp_data {\n+\tuint8_t temp_digests[IMB_MAX_JOBS][DIGEST_LENGTH_MAX];\n+\t/* *< Buffers used to store the digest generated\n+\t * by the driver when verifying a digest provided\n+\t * by the user (using authentication verify operation)\n+\t */\n+};\n+\n+/* Maximum length for digest */\n+#define DIGEST_LENGTH_MAX 64\n+static const unsigned int auth_blocksize[] = {\n+\t\t[IMB_AUTH_NULL]\t\t\t= 0,\n+\t\t[IMB_AUTH_MD5]\t\t\t= 64,\n+\t\t[IMB_AUTH_HMAC_SHA_1]\t\t= 64,\n+\t\t[IMB_AUTH_HMAC_SHA_224]\t\t= 64,\n+\t\t[IMB_AUTH_HMAC_SHA_256]\t\t= 64,\n+\t\t[IMB_AUTH_HMAC_SHA_384]\t\t= 128,\n+\t\t[IMB_AUTH_HMAC_SHA_512]\t\t= 128,\n+\t\t[IMB_AUTH_AES_XCBC]\t\t= 16,\n+\t\t[IMB_AUTH_AES_CCM]\t\t= 16,\n+\t\t[IMB_AUTH_AES_CMAC]\t\t= 16,\n+\t\t[IMB_AUTH_AES_GMAC]\t\t= 16,\n+\t\t[IMB_AUTH_SHA_1]\t\t= 64,\n+\t\t[IMB_AUTH_SHA_224]\t\t= 64,\n+\t\t[IMB_AUTH_SHA_256]\t\t= 64,\n+\t\t[IMB_AUTH_SHA_384]\t\t= 128,\n+\t\t[IMB_AUTH_SHA_512]\t\t= 128,\n+\t\t[IMB_AUTH_ZUC_EIA3_BITLEN]\t= 16,\n+\t\t[IMB_AUTH_SNOW3G_UIA2_BITLEN]\t= 16,\n+\t\t[IMB_AUTH_KASUMI_UIA1]\t\t= 16\n+};\n+\n+/**\n+ * Get the blocksize in bytes for a specified authentication algorithm\n+ *\n+ * @Note: this function will not return a valid value for a non-valid\n+ * authentication algorithm\n+ */\n+static inline unsigned int\n+get_auth_algo_blocksize(IMB_HASH_ALG algo)\n+{\n+\treturn auth_blocksize[algo];\n+}\n+\n+static const unsigned int auth_truncated_digest_byte_lengths[] = {\n+\t\t[IMB_AUTH_MD5]\t\t\t= 12,\n+\t\t[IMB_AUTH_HMAC_SHA_1]\t\t= 12,\n+\t\t[IMB_AUTH_HMAC_SHA_224]\t\t= 14,\n+\t\t[IMB_AUTH_HMAC_SHA_256]\t\t= 16,\n+\t\t[IMB_AUTH_HMAC_SHA_384]\t\t= 24,\n+\t\t[IMB_AUTH_HMAC_SHA_512]\t\t= 32,\n+\t\t[IMB_AUTH_AES_XCBC]\t\t= 12,\n+\t\t[IMB_AUTH_AES_CMAC]\t\t= 12,\n+\t\t[IMB_AUTH_AES_CCM]\t\t= 8,\n+\t\t[IMB_AUTH_NULL]\t\t\t= 0,\n+\t\t[IMB_AUTH_AES_GMAC]\t\t= 12,\n+\t\t[IMB_AUTH_SHA_1]\t\t= 20,\n+\t\t[IMB_AUTH_SHA_224]\t\t= 28,\n+\t\t[IMB_AUTH_SHA_256]\t\t= 32,\n+\t\t[IMB_AUTH_SHA_384]\t\t= 48,\n+\t\t[IMB_AUTH_SHA_512]\t\t= 64,\n+\t\t[IMB_AUTH_ZUC_EIA3_BITLEN]\t= 4,\n+\t\t[IMB_AUTH_SNOW3G_UIA2_BITLEN]\t= 4,\n+\t\t[IMB_AUTH_KASUMI_UIA1]\t\t= 4\n+};\n+\n+/**\n+ * Get the IPsec specified truncated length in bytes of the HMAC digest for a\n+ * specified authentication algorithm\n+ *\n+ * @Note: this function will not return a valid value for a non-valid\n+ * authentication algorithm\n+ */\n+static inline unsigned int\n+get_truncated_digest_byte_length(IMB_HASH_ALG algo)\n+{\n+\treturn auth_truncated_digest_byte_lengths[algo];\n+}\n+\n+static const unsigned int auth_digest_byte_lengths[] = {\n+\t\t[IMB_AUTH_MD5]\t\t\t= 16,\n+\t\t[IMB_AUTH_HMAC_SHA_1]\t\t= 20,\n+\t\t[IMB_AUTH_HMAC_SHA_224]\t\t= 28,\n+\t\t[IMB_AUTH_HMAC_SHA_256]\t\t= 32,\n+\t\t[IMB_AUTH_HMAC_SHA_384]\t\t= 48,\n+\t\t[IMB_AUTH_HMAC_SHA_512]\t\t= 64,\n+\t\t[IMB_AUTH_AES_XCBC]\t\t= 16,\n+\t\t[IMB_AUTH_AES_CMAC]\t\t= 16,\n+\t\t[IMB_AUTH_AES_CCM]\t\t= 16,\n+\t\t[IMB_AUTH_AES_GMAC]\t\t= 16,\n+\t\t[IMB_AUTH_NULL]\t\t\t= 0,\n+\t\t[IMB_AUTH_SHA_1]\t\t= 20,\n+\t\t[IMB_AUTH_SHA_224]\t\t= 28,\n+\t\t[IMB_AUTH_SHA_256]\t\t= 32,\n+\t\t[IMB_AUTH_SHA_384]\t\t= 48,\n+\t\t[IMB_AUTH_SHA_512]\t\t= 64,\n+\t\t[IMB_AUTH_ZUC_EIA3_BITLEN]\t= 4,\n+\t\t[IMB_AUTH_SNOW3G_UIA2_BITLEN]\t= 4,\n+\t\t[IMB_AUTH_KASUMI_UIA1]\t\t= 4\n+\t/**< Vector mode dependent pointer table of the multi-buffer APIs */\n+\n+};\n+\n+/**\n+ * Get the full digest size in bytes for a specified authentication algorithm\n+ * (if available in the Multi-buffer library)\n+ *\n+ * @Note: this function will not return a valid value for a non-valid\n+ * authentication algorithm\n+ */\n+static inline unsigned int\n+get_digest_byte_length(IMB_HASH_ALG algo)\n+{\n+\treturn auth_digest_byte_lengths[algo];\n+}\n+\n+/** AES-NI multi-buffer private session structure */\n+struct aesni_mb_session {\n+\tIMB_CIPHER_MODE cipher_mode;\n+\tIMB_CIPHER_DIRECTION cipher_direction;\n+\tIMB_HASH_ALG hash_alg;\n+\tIMB_CHAIN_ORDER chain_order;\n+\t/*  common job fields */\n+\tstruct {\n+\t\tuint16_t length;\n+\t\tuint16_t offset;\n+\t} iv;\n+\tstruct {\n+\t\tuint16_t length;\n+\t\tuint16_t offset;\n+\t} auth_iv;\n+\t/* *< IV parameters\n+\t */\n+\n+\t/* * Cipher Parameters\n+\t */\n+\tstruct {\n+\t\t/* * Cipher direction - encrypt / decrypt */\n+\t\tIMB_CIPHER_DIRECTION direction;\n+\t\t/* * Cipher mode - CBC / Counter */\n+\t\tIMB_CIPHER_MODE mode;\n+\n+\t\tuint64_t key_length_in_bytes;\n+\n+\t\tunion {\n+\t\t\tstruct {\n+\t\t\t\tuint32_t encode[60] __rte_aligned(16);\n+\t\t\t\t/* *< encode key */\n+\t\t\t\tuint32_t decode[60] __rte_aligned(16);\n+\t\t\t\t/* *< decode key */\n+\t\t\t} expanded_aes_keys;\n+\t\t\t/* *< Expanded AES keys - Allocating space to\n+\t\t\t * contain the maximum expanded key size which\n+\t\t\t * is 240 bytes for 256 bit AES, calculate by:\n+\t\t\t * ((key size (bytes)) *\n+\t\t\t * ((number of rounds) + 1))\n+\t\t\t */\n+\t\t\tstruct {\n+\t\t\t\tconst void *ks_ptr[3];\n+\t\t\t\tuint64_t key[3][16];\n+\t\t\t} exp_3des_keys;\n+\t\t\t/* *< Expanded 3DES keys */\n+\n+\t\t\tstruct gcm_key_data gcm_key;\n+\t\t\t/* *< Expanded GCM key */\n+\t\t\tuint8_t zuc_cipher_key[16];\n+\t\t\t/* *< ZUC cipher key */\n+\t\t\tsnow3g_key_schedule_t pKeySched_snow3g_cipher;\n+\t\t\t/* *< SNOW3G scheduled cipher key */\n+\t\t\tkasumi_key_sched_t pKeySched_kasumi_cipher;\n+\t\t\t/* *< KASUMI scheduled cipher key */\n+\t\t};\n+\t} cipher;\n+\n+\t/* *< Authentication Parameters */\n+\tstruct {\n+\t\tIMB_HASH_ALG algo; /* *< Authentication Algorithm */\n+\t\tenum rte_crypto_auth_operation operation;\n+\t\t/* *< auth operation generate or verify */\n+\t\tunion {\n+\t\t\tstruct {\n+\t\t\t\tuint8_t inner[128] __rte_aligned(16);\n+\t\t\t\t/* *< inner pad */\n+\t\t\t\tuint8_t outer[128] __rte_aligned(16);\n+\t\t\t\t/* *< outer pad */\n+\t\t\t} pads;\n+\t\t\t/* *< HMAC Authentication pads -\n+\t\t\t * allocating space for the maximum pad\n+\t\t\t * size supported which is 128 bytes for\n+\t\t\t * SHA512\n+\t\t\t */\n+\n+\t\t\tstruct {\n+\t\t\t\tuint32_t k1_expanded[44] __rte_aligned(16);\n+\t\t\t\t/* *< k1 (expanded key). */\n+\t\t\t\tuint8_t k2[16] __rte_aligned(16);\n+\t\t\t\t/* *< k2. */\n+\t\t\t\tuint8_t k3[16] __rte_aligned(16);\n+\t\t\t\t/* *< k3. */\n+\t\t\t} xcbc;\n+\n+\t\t\tstruct {\n+\t\t\t\tuint32_t expkey[60] __rte_aligned(16);\n+\t\t\t\t/* *< k1 (expanded key). */\n+\t\t\t\tuint32_t skey1[4] __rte_aligned(16);\n+\t\t\t\t/* *< k2. */\n+\t\t\t\tuint32_t skey2[4] __rte_aligned(16);\n+\t\t\t\t/* *< k3. */\n+\t\t\t} cmac;\n+\t\t\t/* *< Expanded XCBC authentication keys */\n+\t\t\tuint8_t zuc_auth_key[16];\n+\t\t\t/* *< ZUC authentication key */\n+\t\t\tsnow3g_key_schedule_t pKeySched_snow3g_auth;\n+\t\t\t/* *< SNOW3G scheduled authentication key */\n+\t\t\tkasumi_key_sched_t pKeySched_kasumi_auth;\n+\t\t\t/* *< KASUMI scheduled authentication key */\n+\t\t};\n+\t\t/* * Generated digest size by the Multi-buffer library */\n+\t\tuint16_t gen_digest_len;\n+\t\t/* * Requested digest size from Cryptodev */\n+\t\tuint16_t req_digest_len;\n+\n+\t} auth;\n+\tstruct {\n+\t\t/* * AAD data length */\n+\t\tuint16_t aad_len;\n+\t} aead;\n+} __rte_cache_aligned;\n+\n+typedef void (*hash_one_block_t)(const void *data, void *digest);\n+typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys,\n+\t\t\tvoid *dec_exp_keys);\n+\n+\n+/**\n+ * Calculate the authentication pre-computes\n+ *\n+ * @param one_block_hash\tFunction pointer\n+ *\t\t\t\tto calculate digest on ipad/opad\n+ * @param ipad\t\t\tInner pad output byte array\n+ * @param opad\t\t\tOuter pad output byte array\n+ * @param hkey\t\t\tAuthentication key\n+ * @param hkey_len\t\tAuthentication key length\n+ * @param blocksize\t\tBlock size of selected hash algo\n+ */\n+static void\n+calculate_auth_precomputes(hash_one_block_t one_block_hash,\n+\t\tuint8_t *ipad, uint8_t *opad,\n+\t\tconst uint8_t *hkey, uint16_t hkey_len,\n+\t\tuint16_t blocksize)\n+{\n+\tuint32_t i, length;\n+\n+\tuint8_t ipad_buf[blocksize] __rte_aligned(16);\n+\tuint8_t opad_buf[blocksize] __rte_aligned(16);\n+\n+\t/* Setup inner and outer pads */\n+\tmemset(ipad_buf, HMAC_IPAD_VALUE, blocksize);\n+\tmemset(opad_buf, HMAC_OPAD_VALUE, blocksize);\n+\n+\t/* XOR hash key with inner and outer pads */\n+\tlength = hkey_len > blocksize ? blocksize : hkey_len;\n+\n+\tfor (i = 0; i < length; i++) {\n+\t\tipad_buf[i] ^= hkey[i];\n+\t\topad_buf[i] ^= hkey[i];\n+\t}\n+\n+\t/* Compute partial hashes */\n+\t(*one_block_hash)(ipad_buf, ipad);\n+\t(*one_block_hash)(opad_buf, opad);\n+\n+\t/* Clean up stack */\n+\tmemset(ipad_buf, 0, blocksize);\n+\tmemset(opad_buf, 0, blocksize);\n+}\n+\n+static inline int\n+is_aead_algo(IMB_HASH_ALG hash_alg, IMB_CIPHER_MODE cipher_mode)\n+{\n+\treturn (hash_alg == IMB_AUTH_CHACHA20_POLY1305 ||\n+\t\thash_alg == IMB_AUTH_AES_CCM ||\n+\t\t(hash_alg == IMB_AUTH_AES_GMAC &&\n+\t\tcipher_mode == IMB_CIPHER_GCM));\n+}\n+\n+/** Set session authentication parameters */\n+static int\n+aesni_mb_set_session_auth_parameters(const IMB_MGR *mb_mgr,\n+\t\tstruct aesni_mb_session *sess,\n+\t\tconst struct rte_crypto_sym_xform *xform)\n+{\n+\thash_one_block_t hash_oneblock_fn = NULL;\n+\tunsigned int key_larger_block_size = 0;\n+\tuint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };\n+\tuint32_t auth_precompute = 1;\n+\n+\tif (xform == NULL) {\n+\t\tsess->auth.algo = IMB_AUTH_NULL;\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {\n+\t\tIPSEC_MB_LOG(ERR, \"Crypto xform struct not of type auth\");\n+\t\treturn -1;\n+\t}\n+\n+\t/* Set IV parameters */\n+\tsess->auth_iv.offset = xform->auth.iv.offset;\n+\tsess->auth_iv.length = xform->auth.iv.length;\n+\n+\t/* Set the request digest size */\n+\tsess->auth.req_digest_len = xform->auth.digest_length;\n+\n+\t/* Select auth generate/verify */\n+\tsess->auth.operation = xform->auth.op;\n+\n+\t/* Set Authentication Parameters */\n+\tif (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {\n+\t\tsess->auth.algo = IMB_AUTH_AES_XCBC;\n+\n+\t\tuint16_t xcbc_mac_digest_len =\n+\t\t\tget_truncated_digest_byte_length(IMB_AUTH_AES_XCBC);\n+\t\tif (sess->auth.req_digest_len != xcbc_mac_digest_len) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t\tIMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,\n+\t\t\t\tsess->auth.xcbc.k1_expanded,\n+\t\t\t\tsess->auth.xcbc.k2, sess->auth.xcbc.k3);\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {\n+\t\tuint32_t dust[4*15];\n+\n+\t\tsess->auth.algo = IMB_AUTH_AES_CMAC;\n+\n+\t\tuint16_t cmac_digest_len = get_digest_byte_length(IMB_AUTH_AES_CMAC);\n+\n+\t\tif (sess->auth.req_digest_len > cmac_digest_len) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\t/*\n+\t\t * Multi-buffer lib supports digest sizes from 4 to 16 bytes\n+\t\t * in version 0.50 and sizes of 12 and 16 bytes,\n+\t\t * in version 0.49.\n+\t\t * If size requested is different, generate the full digest\n+\t\t * (16 bytes) in a temporary location and then memcpy\n+\t\t * the requested number of bytes.\n+\t\t */\n+\t\tif (sess->auth.req_digest_len < 4)\n+\t\t\tsess->auth.gen_digest_len = cmac_digest_len;\n+\t\telse\n+\t\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,\n+\t\t\t\tsess->auth.cmac.expkey, dust);\n+\t\tIMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,\n+\t\t\t\tsess->auth.cmac.skey1, sess->auth.cmac.skey2);\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {\n+\t\tif (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {\n+\t\t\tsess->cipher.direction = IMB_DIR_ENCRYPT;\n+\t\t\tsess->chain_order = IMB_ORDER_CIPHER_HASH;\n+\t\t} else\n+\t\t\tsess->cipher.direction = IMB_DIR_DECRYPT;\n+\n+\t\tsess->auth.algo = IMB_AUTH_AES_GMAC;\n+\t\tif (sess->auth.req_digest_len >\n+\t\t\tget_digest_byte_length(IMB_AUTH_AES_GMAC)) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\t\tsess->iv.length = xform->auth.iv.length;\n+\t\tsess->iv.offset = xform->auth.iv.offset;\n+\n+\t\tswitch (xform->auth.key.length) {\n+\t\tcase IMB_KEY_128_BYTES:\n+\t\t\tIMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;\n+\t\t\tbreak;\n+\t\tcase IMB_KEY_192_BYTES:\n+\t\t\tIMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;\n+\t\t\tbreak;\n+\t\tcase IMB_KEY_256_BYTES:\n+\t\t\tIMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tRTE_LOG(ERR, PMD, \"failed to parse test type\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {\n+\t\tsess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;\n+\t\tuint16_t zuc_eia3_digest_len =\n+\t\t\tget_truncated_digest_byte_length(\n+\t\t\t\t\t\tIMB_AUTH_ZUC_EIA3_BITLEN);\n+\t\tif (sess->auth.req_digest_len != zuc_eia3_digest_len) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t\tmemcpy(sess->auth.zuc_auth_key, xform->auth.key.data, 16);\n+\t\treturn 0;\n+\t} else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {\n+\t\tsess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN;\n+\t\tuint16_t snow3g_uia2_digest_len =\n+\t\t\tget_truncated_digest_byte_length(\n+\t\t\t\t\t\tIMB_AUTH_SNOW3G_UIA2_BITLEN);\n+\t\tif (sess->auth.req_digest_len != snow3g_uia2_digest_len) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t\tIMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->auth.key.data,\n+\t\t\t\t\t&sess->auth.pKeySched_snow3g_auth);\n+\t\treturn 0;\n+\t} else if (xform->auth.algo == RTE_CRYPTO_AUTH_KASUMI_F9) {\n+\t\tsess->auth.algo = IMB_AUTH_KASUMI_UIA1;\n+\t\tuint16_t kasumi_f9_digest_len =\n+\t\t\tget_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1);\n+\t\tif (sess->auth.req_digest_len != kasumi_f9_digest_len) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t\tIMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr, xform->auth.key.data,\n+\t\t\t\t\t&sess->auth.pKeySched_kasumi_auth);\n+\t\treturn 0;\n+\t}\n+\n+\tswitch (xform->auth.algo) {\n+\tcase RTE_CRYPTO_AUTH_MD5_HMAC:\n+\t\tsess->auth.algo = IMB_AUTH_MD5;\n+\t\thash_oneblock_fn = mb_mgr->md5_one_block;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA1_HMAC:\n+\t\tsess->auth.algo = IMB_AUTH_HMAC_SHA_1;\n+\t\thash_oneblock_fn = mb_mgr->sha1_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(IMB_AUTH_HMAC_SHA_1)) {\n+\t\t\tIMB_SHA1(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA1:\n+\t\tsess->auth.algo = IMB_AUTH_SHA_1;\n+\t\tauth_precompute = 0;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA224_HMAC:\n+\t\tsess->auth.algo = IMB_AUTH_HMAC_SHA_224;\n+\t\thash_oneblock_fn = mb_mgr->sha224_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(IMB_AUTH_HMAC_SHA_224)) {\n+\t\t\tIMB_SHA224(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA224:\n+\t\tsess->auth.algo = IMB_AUTH_SHA_224;\n+\t\tauth_precompute = 0;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA256_HMAC:\n+\t\tsess->auth.algo = IMB_AUTH_HMAC_SHA_256;\n+\t\thash_oneblock_fn = mb_mgr->sha256_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(IMB_AUTH_HMAC_SHA_256)) {\n+\t\t\tIMB_SHA256(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA256:\n+\t\tsess->auth.algo = IMB_AUTH_SHA_256;\n+\t\tauth_precompute = 0;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA384_HMAC:\n+\t\tsess->auth.algo = IMB_AUTH_HMAC_SHA_384;\n+\t\thash_oneblock_fn = mb_mgr->sha384_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(IMB_AUTH_HMAC_SHA_384)) {\n+\t\t\tIMB_SHA384(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA384:\n+\t\tsess->auth.algo = IMB_AUTH_SHA_384;\n+\t\tauth_precompute = 0;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA512_HMAC:\n+\t\tsess->auth.algo = IMB_AUTH_HMAC_SHA_512;\n+\t\thash_oneblock_fn = mb_mgr->sha512_one_block;\n+\t\tif (xform->auth.key.length > get_auth_algo_blocksize(IMB_AUTH_HMAC_SHA_512)) {\n+\t\t\tIMB_SHA512(mb_mgr,\n+\t\t\t\txform->auth.key.data,\n+\t\t\t\txform->auth.key.length,\n+\t\t\t\thashed_key);\n+\t\t\tkey_larger_block_size = 1;\n+\t\t}\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AUTH_SHA512:\n+\t\tsess->auth.algo = IMB_AUTH_SHA_512;\n+\t\tauth_precompute = 0;\n+\t\tbreak;\n+\tdefault:\n+\t\tIPSEC_MB_LOG(ERR,\n+\t\t\t\"Unsupported authentication algorithm selection\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\tuint16_t trunc_digest_size =\n+\t\t\tget_truncated_digest_byte_length(sess->auth.algo);\n+\tuint16_t full_digest_size =\n+\t\t\tget_digest_byte_length(sess->auth.algo);\n+\n+\tif (sess->auth.req_digest_len > full_digest_size ||\n+\t\t\tsess->auth.req_digest_len == 0) {\n+\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (sess->auth.req_digest_len != trunc_digest_size &&\n+\t\t\tsess->auth.req_digest_len != full_digest_size)\n+\t\tsess->auth.gen_digest_len = full_digest_size;\n+\telse\n+\t\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\t/* Plain SHA does not require precompute key */\n+\tif (auth_precompute == 0)\n+\t\treturn 0;\n+\n+\t/* Calculate Authentication precomputes */\n+\tif (key_larger_block_size) {\n+\t\tcalculate_auth_precomputes(hash_oneblock_fn,\n+\t\t\tsess->auth.pads.inner, sess->auth.pads.outer,\n+\t\t\thashed_key,\n+\t\t\txform->auth.key.length,\n+\t\t\tget_auth_algo_blocksize(sess->auth.algo));\n+\t} else {\n+\t\tcalculate_auth_precomputes(hash_oneblock_fn,\n+\t\t\tsess->auth.pads.inner, sess->auth.pads.outer,\n+\t\t\txform->auth.key.data,\n+\t\t\txform->auth.key.length,\n+\t\t\tget_auth_algo_blocksize(sess->auth.algo));\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/** Set session cipher parameters */\n+static int\n+aesni_mb_set_session_cipher_parameters(const IMB_MGR *mb_mgr,\n+\t\tstruct aesni_mb_session *sess,\n+\t\tconst struct rte_crypto_sym_xform *xform)\n+{\n+\tuint8_t is_aes = 0;\n+\tuint8_t is_3DES = 0;\n+\tuint8_t is_docsis = 0;\n+\tuint8_t is_zuc = 0;\n+\tuint8_t is_snow3g = 0;\n+\tuint8_t is_kasumi = 0;\n+\n+\tif (xform == NULL) {\n+\t\tsess->cipher.mode = IMB_CIPHER_NULL;\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {\n+\t\tIPSEC_MB_LOG(ERR, \"Crypto xform struct not of type cipher\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* Select cipher direction */\n+\tswitch (xform->cipher.op) {\n+\tcase RTE_CRYPTO_CIPHER_OP_ENCRYPT:\n+\t\tsess->cipher.direction = IMB_DIR_ENCRYPT;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_OP_DECRYPT:\n+\t\tsess->cipher.direction = IMB_DIR_DECRYPT;\n+\t\tbreak;\n+\tdefault:\n+\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher operation parameter\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* Select cipher mode */\n+\tswitch (xform->cipher.algo) {\n+\tcase RTE_CRYPTO_CIPHER_AES_CBC:\n+\t\tsess->cipher.mode = IMB_CIPHER_CBC;\n+\t\tis_aes = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_AES_CTR:\n+\t\tsess->cipher.mode = IMB_CIPHER_CNTR;\n+\t\tis_aes = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_AES_DOCSISBPI:\n+\t\tsess->cipher.mode = IMB_CIPHER_DOCSIS_SEC_BPI;\n+\t\tis_docsis = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_DES_CBC:\n+\t\tsess->cipher.mode = IMB_CIPHER_DES;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_DES_DOCSISBPI:\n+\t\tsess->cipher.mode = IMB_CIPHER_DOCSIS_DES;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_3DES_CBC:\n+\t\tsess->cipher.mode = IMB_CIPHER_DES3;\n+\t\tis_3DES = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_AES_ECB:\n+\t\tsess->cipher.mode = IMB_CIPHER_ECB;\n+\t\tis_aes = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_ZUC_EEA3:\n+\t\tsess->cipher.mode = IMB_CIPHER_ZUC_EEA3;\n+\t\tis_zuc = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_SNOW3G_UEA2:\n+\t\tsess->cipher.mode = IMB_CIPHER_SNOW3G_UEA2_BITLEN;\n+\t\tis_snow3g = 1;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_CIPHER_KASUMI_F8:\n+\t\tsess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN;\n+\t\tis_kasumi = 1;\n+\t\tbreak;\n+\tdefault:\n+\t\tIPSEC_MB_LOG(ERR, \"Unsupported cipher mode parameter\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* Set IV parameters */\n+\tsess->iv.offset = xform->cipher.iv.offset;\n+\tsess->iv.length = xform->cipher.iv.length;\n+\n+\t/* Check key length and choose key expansion function for AES */\n+\tif (is_aes) {\n+\t\tswitch (xform->cipher.key.length) {\n+\t\tcase IMB_KEY_128_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;\n+\t\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tcase IMB_KEY_192_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;\n+\t\t\tIMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tcase IMB_KEY_256_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;\n+\t\t\tIMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t} else if (is_docsis) {\n+\t\tswitch (xform->cipher.key.length) {\n+\t\tcase IMB_KEY_128_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;\n+\t\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tcase IMB_KEY_256_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;\n+\t\t\tIMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t} else if (is_3DES) {\n+\t\tuint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],\n+\t\t\t\tsess->cipher.exp_3des_keys.key[1],\n+\t\t\t\tsess->cipher.exp_3des_keys.key[2]};\n+\n+\t\tswitch (xform->cipher.key.length) {\n+\t\tcase  24:\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n+\t\t\t\t\txform->cipher.key.data);\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[1],\n+\t\t\t\t\txform->cipher.key.data + 8);\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[2],\n+\t\t\t\t\txform->cipher.key.data + 16);\n+\n+\t\t\t/* Initialize keys - 24 bytes: [K1-K2-K3] */\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];\n+\t\t\tbreak;\n+\t\tcase 16:\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n+\t\t\t\t\txform->cipher.key.data);\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[1],\n+\t\t\t\t\txform->cipher.key.data + 8);\n+\t\t\t/* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];\n+\t\t\tbreak;\n+\t\tcase 8:\n+\t\t\tIMB_DES_KEYSCHED(mb_mgr, keys[0],\n+\t\t\t\t\txform->cipher.key.data);\n+\n+\t\t\t/* Initialize keys - 8 bytes: [K1 = K2 = K3] */\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];\n+\t\t\tsess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\tsess->cipher.key_length_in_bytes = 24;\n+\t} else if (is_zuc) {\n+\t\tif (xform->cipher.key.length != 16) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->cipher.key_length_in_bytes = 16;\n+\t\tmemcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data,\n+\t\t\t16);\n+\t} else if (is_snow3g) {\n+\t\tif (xform->cipher.key.length != 16) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->cipher.key_length_in_bytes = 16;\n+\t\tIMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\t&sess->cipher.pKeySched_snow3g_cipher);\n+\t} else if (is_kasumi) {\n+\t\tif (xform->cipher.key.length != 16) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->cipher.key_length_in_bytes = 16;\n+\t\tIMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data,\n+\t\t\t\t\t&sess->cipher.pKeySched_kasumi_cipher);\n+\t} else {\n+\t\tif (xform->cipher.key.length != 8) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->cipher.key_length_in_bytes = 8;\n+\n+\t\tIMB_DES_KEYSCHED(mb_mgr,\n+\t\t\t(uint64_t *)sess->cipher.expanded_aes_keys.encode,\n+\t\t\t\txform->cipher.key.data);\n+\t\tIMB_DES_KEYSCHED(mb_mgr,\n+\t\t\t(uint64_t *)sess->cipher.expanded_aes_keys.decode,\n+\t\t\t\txform->cipher.key.data);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int\n+aesni_mb_set_session_aead_parameters(const IMB_MGR *mb_mgr,\n+\t\tstruct aesni_mb_session *sess,\n+\t\tconst struct rte_crypto_sym_xform *xform)\n+{\n+\tswitch (xform->aead.op) {\n+\tcase RTE_CRYPTO_AEAD_OP_ENCRYPT:\n+\t\tsess->cipher.direction = IMB_DIR_ENCRYPT;\n+\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;\n+\t\tbreak;\n+\tcase RTE_CRYPTO_AEAD_OP_DECRYPT:\n+\t\tsess->cipher.direction = IMB_DIR_DECRYPT;\n+\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;\n+\t\tbreak;\n+\tdefault:\n+\t\tIPSEC_MB_LOG(ERR, \"Invalid aead operation parameter\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* Set IV parameters */\n+\tsess->iv.offset = xform->aead.iv.offset;\n+\tsess->iv.length = xform->aead.iv.length;\n+\n+\t/* Set digest sizes */\n+\tsess->auth.req_digest_len = xform->aead.digest_length;\n+\tsess->auth.gen_digest_len = sess->auth.req_digest_len;\n+\n+\tswitch (xform->aead.algo) {\n+\tcase RTE_CRYPTO_AEAD_AES_CCM:\n+\t\tsess->cipher.mode = IMB_CIPHER_CCM;\n+\t\tsess->auth.algo = IMB_AUTH_AES_CCM;\n+\n+\t\t/* Check key length and choose key expansion function for AES */\n+\t\tswitch (xform->aead.key.length) {\n+\t\tcase IMB_KEY_128_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;\n+\t\t\tIMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tcase IMB_KEY_256_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;\n+\t\t\tIMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.encode,\n+\t\t\t\t\tsess->cipher.expanded_aes_keys.decode);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\t/* CCM digests must be between 4 and 16 and an even number */\n+\t\tif (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||\n+\t\t\tsess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||\n+\t\t\t(sess->auth.req_digest_len & 1) == 1) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tbreak;\n+\n+\tcase RTE_CRYPTO_AEAD_AES_GCM:\n+\t\tsess->cipher.mode = IMB_CIPHER_GCM;\n+\t\tsess->auth.algo = IMB_AUTH_AES_GMAC;\n+\n+\t\tswitch (xform->aead.key.length) {\n+\t\tcase IMB_KEY_128_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;\n+\t\t\tIMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tbreak;\n+\t\tcase IMB_KEY_192_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;\n+\t\t\tIMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tbreak;\n+\t\tcase IMB_KEY_256_BYTES:\n+\t\t\tsess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;\n+\t\t\tIMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,\n+\t\t\t\t&sess->cipher.gcm_key);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid cipher key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\t/* GCM digest size must be between 1 and 16 */\n+\t\tif (sess->auth.req_digest_len == 0 ||\n+\t\t\t\tsess->auth.req_digest_len > 16) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tbreak;\n+\n+\tcase RTE_CRYPTO_AEAD_CHACHA20_POLY1305:\n+\t\tsess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305;\n+\t\tsess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;\n+\n+\t\tif (xform->aead.key.length != 32) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid key length\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tsess->cipher.key_length_in_bytes = 32;\n+\t\tmemcpy(sess->cipher.expanded_aes_keys.encode,\n+\t\t\txform->aead.key.data, 32);\n+\t\tif (sess->auth.req_digest_len != 16) {\n+\t\t\tIPSEC_MB_LOG(ERR, \"Invalid digest size\\n\");\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tbreak;\n+\tdefault:\n+\t\tIPSEC_MB_LOG(ERR, \"Unsupported aead mode parameter\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/** Configure a aesni multi-buffer session from a crypto xform chain */\n+static int\n+aesni_mb_session_configure(IMB_MGR *mb_mgr,\n+\t\tvoid *priv_sess,\n+\t\tconst struct rte_crypto_sym_xform *xform)\n+{\n+\tconst struct rte_crypto_sym_xform *auth_xform = NULL;\n+\tconst struct rte_crypto_sym_xform *cipher_xform = NULL;\n+\tconst struct rte_crypto_sym_xform *aead_xform = NULL;\n+\tenum ipsec_mb_operation mode;\n+\tstruct aesni_mb_session *sess = (struct aesni_mb_session *) priv_sess;\n+\tint ret;\n+\n+\tret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,\n+\t\t\t\t&cipher_xform, &aead_xform);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\t/* Select Crypto operation - hash then cipher / cipher then hash */\n+\tswitch (mode) {\n+\tcase IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:\n+\t\tsess->chain_order = IMB_ORDER_HASH_CIPHER;\n+\t\tbreak;\n+\tcase IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:\n+\tcase IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:\n+\t\tsess->chain_order = IMB_ORDER_CIPHER_HASH;\n+\t\tbreak;\n+\tcase IPSEC_MB_OP_HASH_GEN_ONLY:\n+\tcase IPSEC_MB_OP_HASH_VERIFY_ONLY:\n+\tcase IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:\n+\t\tsess->chain_order = IMB_ORDER_HASH_CIPHER;\n+\t\tbreak;\n+\t/*\n+\t * Multi buffer library operates only at two modes,\n+\t * IMB_ORDER_CIPHER_HASH and IMB_ORDER_HASH_CIPHER.\n+\t * When doing ciphering only, chain order depends\n+\t * on cipher operation: encryption is always\n+\t * the first operation and decryption the last one.\n+\t */\n+\tcase IPSEC_MB_OP_ENCRYPT_ONLY:\n+\t\tsess->chain_order = IMB_ORDER_CIPHER_HASH;\n+\t\tbreak;\n+\tcase IPSEC_MB_OP_DECRYPT_ONLY:\n+\t\tsess->chain_order = IMB_ORDER_HASH_CIPHER;\n+\t\tbreak;\n+\tcase IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:\n+\t\tsess->chain_order = IMB_ORDER_CIPHER_HASH;\n+\t\tsess->aead.aad_len = xform->aead.aad_length;\n+\t\tbreak;\n+\tcase IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:\n+\t\tsess->chain_order = IMB_ORDER_HASH_CIPHER;\n+\t\tsess->aead.aad_len = xform->aead.aad_length;\n+\t\tbreak;\n+\tcase IPSEC_MB_OP_NOT_SUPPORTED:\n+\tdefault:\n+\t\tIPSEC_MB_LOG(ERR,\n+\t\t\t\"Unsupported operation chain order parameter\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* Default IV length = 0 */\n+\tsess->iv.length = 0;\n+\tsess->auth_iv.length = 0;\n+\n+\tret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);\n+\tif (ret != 0) {\n+\t\tIPSEC_MB_LOG(ERR,\n+\t\t\t\"Invalid/unsupported authentication parameters\");\n+\t\treturn ret;\n+\t}\n+\n+\tret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,\n+\t\t\tcipher_xform);\n+\tif (ret != 0) {\n+\t\tIPSEC_MB_LOG(ERR, \"Invalid/unsupported cipher parameters\");\n+\t\treturn ret;\n+\t}\n+\n+\tif (aead_xform) {\n+\t\tret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,\n+\t\t\t\taead_xform);\n+\t\tif (ret != 0) {\n+\t\t\tIPSEC_MB_LOG(ERR,\n+\t\t\t\t\"Invalid/unsupported aead parameters\");\n+\t\t\treturn ret;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n+/** Check DOCSIS security session configuration is valid */\n+static int\n+check_docsis_sec_session(struct rte_security_session_conf *conf)\n+{\n+\tstruct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;\n+\tstruct rte_security_docsis_xform *docsis = &conf->docsis;\n+\n+\t/* Downlink: CRC generate -> Cipher encrypt */\n+\tif (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {\n+\n+\t\tif (crypto_sym != NULL &&\n+\t\t    crypto_sym->type ==\tRTE_CRYPTO_SYM_XFORM_CIPHER &&\n+\t\t    crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&\n+\t\t    crypto_sym->cipher.algo ==\n+\t\t\t\t\tRTE_CRYPTO_CIPHER_AES_DOCSISBPI &&\n+\t\t    (crypto_sym->cipher.key.length == IMB_KEY_128_BYTES ||\n+\t\t     crypto_sym->cipher.key.length == IMB_KEY_256_BYTES) &&\n+\t\t    crypto_sym->cipher.iv.length == IMB_AES_BLOCK_SIZE &&\n+\t\t    crypto_sym->next == NULL) {\n+\t\t\treturn 0;\n+\t\t}\n+\t/* Uplink: Cipher decrypt -> CRC verify */\n+\t} else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {\n+\n+\t\tif (crypto_sym != NULL &&\n+\t\t    crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&\n+\t\t    crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&\n+\t\t    crypto_sym->cipher.algo ==\n+\t\t\t\t\tRTE_CRYPTO_CIPHER_AES_DOCSISBPI &&\n+\t\t    (crypto_sym->cipher.key.length == IMB_KEY_128_BYTES ||\n+\t\t     crypto_sym->cipher.key.length == IMB_KEY_256_BYTES) &&\n+\t\t    crypto_sym->cipher.iv.length == IMB_AES_BLOCK_SIZE &&\n+\t\t    crypto_sym->next == NULL) {\n+\t\t\treturn 0;\n+\t\t}\n+\t}\n+\n+\treturn -EINVAL;\n+}\n+\n+/** Set DOCSIS security session auth (CRC) parameters */\n+static int\n+aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,\n+\t\tstruct rte_security_docsis_xform *xform)\n+{\n+\tif (xform == NULL) {\n+\t\tIPSEC_MB_LOG(ERR, \"Invalid DOCSIS xform\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* Select CRC generate/verify */\n+\tif (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) {\n+\t\tsess->auth.algo = IMB_AUTH_DOCSIS_CRC32;\n+\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;\n+\t} else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {\n+\t\tsess->auth.algo = IMB_AUTH_DOCSIS_CRC32;\n+\t\tsess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;\n+\t} else {\n+\t\tIPSEC_MB_LOG(ERR, \"Unsupported DOCSIS direction\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\tsess->auth.req_digest_len = RTE_ETHER_CRC_LEN;\n+\tsess->auth.gen_digest_len = RTE_ETHER_CRC_LEN;\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * Parse DOCSIS security session configuration and set private session\n+ * parameters\n+ */\n+static int\n+aesni_mb_set_docsis_sec_session_parameters(\n+\t\t__rte_unused struct rte_cryptodev *dev,\n+\t\tstruct rte_security_session_conf *conf,\n+\t\tvoid *sess)\n+{\n+\tIMB_MGR  *mb_mgr = alloc_init_mb_mgr();\n+\tstruct rte_security_docsis_xform *docsis_xform;\n+\tstruct rte_crypto_sym_xform *cipher_xform;\n+\tstruct aesni_mb_session *ipsec_sess = sess;\n+\tint ret = 0;\n+\n+\tif (!mb_mgr)\n+\t\treturn -ENOMEM;\n+\n+\tret = check_docsis_sec_session(conf);\n+\tif (ret) {\n+\t\tIPSEC_MB_LOG(ERR, \"Unsupported DOCSIS security configuration\");\n+\t\tgoto error_exit;\n+\t}\n+\n+\tswitch (conf->docsis.direction) {\n+\tcase RTE_SECURITY_DOCSIS_UPLINK:\n+\t\tipsec_sess->chain_order = IMB_ORDER_CIPHER_HASH;\n+\t\tdocsis_xform = &conf->docsis;\n+\t\tcipher_xform = conf->crypto_xform;\n+\t\tbreak;\n+\tcase RTE_SECURITY_DOCSIS_DOWNLINK:\n+\t\tipsec_sess->chain_order = IMB_ORDER_HASH_CIPHER;\n+\t\tcipher_xform = conf->crypto_xform;\n+\t\tdocsis_xform = &conf->docsis;\n+\t\tbreak;\n+\tdefault:\n+\t\tIPSEC_MB_LOG(ERR, \"Unsupported DOCSIS security configuration\");\n+\t\tret = -EINVAL;\n+\t\tgoto error_exit;\n+\t}\n+\n+\t/* Default IV length = 0 */\n+\tipsec_sess->iv.length = 0;\n+\n+\tret = aesni_mb_set_docsis_sec_session_auth_parameters(ipsec_sess,\n+\t\t\tdocsis_xform);\n+\tif (ret != 0) {\n+\t\tIPSEC_MB_LOG(ERR, \"Invalid/unsupported DOCSIS parameters\");\n+\t\tgoto error_exit;\n+\t}\n+\n+\tret = aesni_mb_set_session_cipher_parameters(mb_mgr,\n+\t\t\tipsec_sess, cipher_xform);\n+\n+\tif (ret != 0) {\n+\t\tIPSEC_MB_LOG(ERR, \"Invalid/unsupported cipher parameters\");\n+\t\tgoto error_exit;\n+\t}\n+\n+error_exit:\n+\tfree_mb_mgr(mb_mgr);\n+\treturn ret;\n+}\n+#endif\n+\n+static inline uint64_t\n+auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,\n+\t\tuint32_t oop)\n+{\n+\tstruct rte_mbuf *m_src, *m_dst;\n+\tuint8_t *p_src, *p_dst;\n+\tuintptr_t u_src, u_dst;\n+\tuint32_t cipher_end, auth_end;\n+\n+\t/* Only cipher then hash needs special calculation. */\n+\tif (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH)\n+\t\treturn op->sym->auth.data.offset;\n+\n+\tm_src = op->sym->m_src;\n+\tm_dst = op->sym->m_dst;\n+\n+\tp_src = rte_pktmbuf_mtod(m_src, uint8_t *);\n+\tp_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);\n+\tu_src = (uintptr_t)p_src;\n+\tu_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;\n+\n+\t/**\n+\t * Copy the content between cipher offset and auth offset for generating\n+\t * correct digest.\n+\t */\n+\tif (op->sym->cipher.data.offset > op->sym->auth.data.offset)\n+\t\tmemcpy(p_dst + op->sym->auth.data.offset,\n+\t\t\t\tp_src + op->sym->auth.data.offset,\n+\t\t\t\top->sym->cipher.data.offset -\n+\t\t\t\top->sym->auth.data.offset);\n+\n+\t/**\n+\t * Copy the content between (cipher offset + length) and (auth offset +\n+\t * length) for generating correct digest\n+\t */\n+\tcipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;\n+\tauth_end = op->sym->auth.data.offset + op->sym->auth.data.length;\n+\tif (cipher_end < auth_end)\n+\t\tmemcpy(p_dst + cipher_end, p_src + cipher_end,\n+\t\t\t\tauth_end - cipher_end);\n+\n+\t/**\n+\t * Since intel-ipsec-mb only supports positive values,\n+\t * we need to deduct the correct offset between src and dst.\n+\t */\n+\n+\treturn u_src < u_dst ? (u_dst - u_src) :\n+\t\t\t(UINT64_MAX - u_src + u_dst + 1);\n+}\n+\n+static inline void\n+set_cpu_mb_job_params(IMB_JOB *job, struct aesni_mb_session *session,\n+\t\tunion rte_crypto_sym_ofs sofs, void *buf, uint32_t len,\n+\t\tstruct rte_crypto_va_iova_ptr *iv,\n+\t\tstruct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)\n+{\n+\t/* Set crypto operation */\n+\tjob->chain_order = session->chain_order;\n+\n+\t/* Set cipher parameters */\n+\tjob->cipher_direction = session->cipher.direction;\n+\tjob->cipher_mode = session->cipher.mode;\n+\n+\tjob->key_len_in_bytes = session->cipher.key_length_in_bytes;\n+\n+\t/* Set authentication parameters */\n+\tjob->hash_alg = session->auth.algo;\n+\tjob->iv = iv->va;\n+\n+\tswitch (job->hash_alg) {\n+\tcase IMB_AUTH_AES_XCBC:\n+\t\tjob->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;\n+\t\tjob->u.XCBC._k2 = session->auth.xcbc.k2;\n+\t\tjob->u.XCBC._k3 = session->auth.xcbc.k3;\n+\n+\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_AES_CCM:\n+\t\tjob->u.CCM.aad = (uint8_t *)aad->va + 18;\n+\t\tjob->u.CCM.aad_len_in_bytes = session->aead.aad_len;\n+\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\t\tjob->iv++;\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_AES_CMAC:\n+\t\tjob->u.CMAC._key_expanded = session->auth.cmac.expkey;\n+\t\tjob->u.CMAC._skey1 = session->auth.cmac.skey1;\n+\t\tjob->u.CMAC._skey2 = session->auth.cmac.skey2;\n+\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_AES_GMAC:\n+\t\tif (session->cipher.mode == IMB_CIPHER_GCM) {\n+\t\t\tjob->u.GCM.aad = aad->va;\n+\t\t\tjob->u.GCM.aad_len_in_bytes = session->aead.aad_len;\n+\t\t} else {\n+\t\t\t/* For GMAC */\n+\t\t\tjob->u.GCM.aad = buf;\n+\t\t\tjob->u.GCM.aad_len_in_bytes = len;\n+\t\t\tjob->cipher_mode = IMB_CIPHER_GCM;\n+\t\t}\n+\t\tjob->enc_keys = &session->cipher.gcm_key;\n+\t\tjob->dec_keys = &session->cipher.gcm_key;\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_CHACHA20_POLY1305:\n+\t\tjob->u.CHACHA20_POLY1305.aad = aad->va;\n+\t\tjob->u.CHACHA20_POLY1305.aad_len_in_bytes =\n+\t\t\tsession->aead.aad_len;\n+\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tjob->dec_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tbreak;\n+\tdefault:\n+\t\tjob->u.HMAC._hashed_auth_key_xor_ipad =\n+\t\t\t\tsession->auth.pads.inner;\n+\t\tjob->u.HMAC._hashed_auth_key_xor_opad =\n+\t\t\t\tsession->auth.pads.outer;\n+\n+\t\tif (job->cipher_mode == IMB_CIPHER_DES3) {\n+\t\t\tjob->enc_keys = session->cipher.exp_3des_keys.ks_ptr;\n+\t\t\tjob->dec_keys = session->cipher.exp_3des_keys.ks_ptr;\n+\t\t} else {\n+\t\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\t\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\t\t}\n+\t}\n+\n+\t/*\n+\t * Multi-buffer library current only support returning a truncated\n+\t * digest length as specified in the relevant IPsec RFCs\n+\t */\n+\n+\t/* Set digest location and length */\n+\tjob->auth_tag_output = digest;\n+\tjob->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;\n+\n+\t/* Set IV parameters */\n+\tjob->iv_len_in_bytes = session->iv.length;\n+\n+\t/* Data Parameters */\n+\tjob->src = buf;\n+\tjob->dst = (uint8_t *)buf + sofs.ofs.cipher.head;\n+\tjob->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;\n+\tjob->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;\n+\tif (job->hash_alg == IMB_AUTH_AES_GMAC && session->cipher.mode != IMB_CIPHER_GCM) {\n+\t\tjob->msg_len_to_hash_in_bytes = 0;\n+\t\tjob->msg_len_to_cipher_in_bytes = 0;\n+\t} else {\n+\t\tjob->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -\n+\t\t\tsofs.ofs.auth.tail;\n+\t\tjob->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -\n+\t\t\tsofs.ofs.cipher.tail;\n+\t}\n+\n+\tjob->user_data = udata;\n+}\n+\n+/**\n+ * Process a crypto operation and complete a IMB_JOB job structure for\n+ * submission to the multi buffer library for processing.\n+ *\n+ * @param\tqp\t\tqueue pair\n+ * @param\tjob\t\tIMB_JOB structure to fill\n+ * @param\top\t\tcrypto op to process\n+ * @param\tdigest_idx\tID for digest to use\n+ *\n+ * @return\n+ * - 0 on success, the IMB_JOB will be filled\n+ * - -1 if invalid session, IMB_JOB will not be filled\n+ */\n+static inline int\n+set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,\n+\t\tstruct rte_crypto_op *op, uint8_t *digest_idx)\n+{\n+\tstruct rte_mbuf *m_src = op->sym->m_src, *m_dst;\n+\tstruct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);\n+\tstruct aesni_mb_session *session;\n+\tuint32_t m_offset, oop;\n+\n+\tsession = ipsec_mb_get_session_private(qp, op);\n+\tif (session == NULL) {\n+\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\t\treturn -1;\n+\t}\n+\n+\t/* Set crypto operation */\n+\tjob->chain_order = session->chain_order;\n+\n+\t/* Set cipher parameters */\n+\tjob->cipher_direction = session->cipher.direction;\n+\tjob->cipher_mode = session->cipher.mode;\n+\n+\tjob->key_len_in_bytes = session->cipher.key_length_in_bytes;\n+\n+\t/* Set authentication parameters */\n+\tjob->hash_alg = session->auth.algo;\n+\n+\tconst int aead = is_aead_algo(job->hash_alg, job->cipher_mode);\n+\n+\tswitch (job->hash_alg) {\n+\tcase IMB_AUTH_AES_XCBC:\n+\t\tjob->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;\n+\t\tjob->u.XCBC._k2 = session->auth.xcbc.k2;\n+\t\tjob->u.XCBC._k3 = session->auth.xcbc.k3;\n+\n+\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_AES_CCM:\n+\t\tjob->u.CCM.aad = op->sym->aead.aad.data + 18;\n+\t\tjob->u.CCM.aad_len_in_bytes = session->aead.aad_len;\n+\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_AES_CMAC:\n+\t\tjob->u.CMAC._key_expanded = session->auth.cmac.expkey;\n+\t\tjob->u.CMAC._skey1 = session->auth.cmac.skey1;\n+\t\tjob->u.CMAC._skey2 = session->auth.cmac.skey2;\n+\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_AES_GMAC:\n+\t\tif (session->cipher.mode == IMB_CIPHER_GCM) {\n+\t\t\tjob->u.GCM.aad = op->sym->aead.aad.data;\n+\t\t\tjob->u.GCM.aad_len_in_bytes = session->aead.aad_len;\n+\t\t} else {\n+\t\t\t/* For GMAC */\n+\t\t\tjob->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,\n+\t\t\t\t\tuint8_t *, op->sym->auth.data.offset);\n+\t\t\tjob->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;\n+\t\t\tjob->cipher_mode = IMB_CIPHER_GCM;\n+\t\t}\n+\t\tjob->enc_keys = &session->cipher.gcm_key;\n+\t\tjob->dec_keys = &session->cipher.gcm_key;\n+\t\tbreak;\n+\tcase IMB_AUTH_ZUC_EIA3_BITLEN:\n+\t\tjob->u.ZUC_EIA3._key = session->auth.zuc_auth_key;\n+\t\tjob->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\t\t\t\tsession->auth_iv.offset);\n+\t\tbreak;\n+\tcase IMB_AUTH_SNOW3G_UIA2_BITLEN:\n+\t\tjob->u.SNOW3G_UIA2._key = (void *)\n+\t\t\t&session->auth.pKeySched_snow3g_auth;\n+\t\tjob->u.SNOW3G_UIA2._iv =\n+\t\t\trte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\t\t\t\tsession->auth_iv.offset);\n+\t\tbreak;\n+\tcase IMB_AUTH_KASUMI_UIA1:\n+\t\tjob->u.KASUMI_UIA1._key = (void *)\n+\t\t\t&session->auth.pKeySched_kasumi_auth;\n+\t\tbreak;\n+\tcase IMB_AUTH_CHACHA20_POLY1305:\n+\t\tjob->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;\n+\t\tjob->u.CHACHA20_POLY1305.aad_len_in_bytes =\n+\t\t\tsession->aead.aad_len;\n+\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tjob->dec_keys = session->cipher.expanded_aes_keys.encode;\n+\t\tbreak;\n+\tdefault:\n+\t\tjob->u.HMAC._hashed_auth_key_xor_ipad =\n+\t\t\tsession->auth.pads.inner;\n+\t\tjob->u.HMAC._hashed_auth_key_xor_opad =\n+\t\t\tsession->auth.pads.outer;\n+\n+\t\tif (job->cipher_mode == IMB_CIPHER_DES3) {\n+\t\t\tjob->enc_keys = session->cipher.exp_3des_keys.ks_ptr;\n+\t\t\tjob->dec_keys = session->cipher.exp_3des_keys.ks_ptr;\n+\t\t} else {\n+\t\t\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\t\t\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\t\t}\n+\t}\n+\n+\tif (aead)\n+\t\tm_offset = op->sym->aead.data.offset;\n+\telse\n+\t\tm_offset = op->sym->cipher.data.offset;\n+\n+\tif (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {\n+\t\tjob->enc_keys = session->cipher.zuc_cipher_key;\n+\t\tjob->dec_keys = session->cipher.zuc_cipher_key;\n+\t} else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {\n+\t\tjob->enc_keys = &session->cipher.pKeySched_snow3g_cipher;\n+\t\tm_offset = 0;\n+\t} else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) {\n+\t\tjob->enc_keys = &session->cipher.pKeySched_kasumi_cipher;\n+\t\tm_offset = 0;\n+\t}\n+\n+\tif (!op->sym->m_dst) {\n+\t\t/* in-place operation */\n+\t\tm_dst = m_src;\n+\t\toop = 0;\n+\t} else if (op->sym->m_dst == op->sym->m_src) {\n+\t\t/* in-place operation */\n+\t\tm_dst = m_src;\n+\t\toop = 0;\n+\t} else {\n+\t\t/* out-of-place operation */\n+\t\tm_dst = op->sym->m_dst;\n+\t\toop = 1;\n+\t}\n+\n+\t/* Set digest output location */\n+\tif (job->hash_alg != IMB_AUTH_NULL &&\n+\t\t\tsession->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {\n+\t\tjob->auth_tag_output = qp_data->temp_digests[*digest_idx];\n+\t\t*digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;\n+\t} else {\n+\t\tif (aead)\n+\t\t\tjob->auth_tag_output = op->sym->aead.digest.data;\n+\t\telse\n+\t\t\tjob->auth_tag_output = op->sym->auth.digest.data;\n+\n+\t\tif (session->auth.req_digest_len !=\n+\t\t\t\tsession->auth.gen_digest_len) {\n+\t\t\tjob->auth_tag_output =\n+\t\t\t\tqp_data->temp_digests[*digest_idx];\n+\t\t\t*digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;\n+\t\t}\n+\t}\n+\t/*\n+\t * Multi-buffer library current only support returning a truncated\n+\t * digest length as specified in the relevant IPsec RFCs\n+\t */\n+\n+\t/* Set digest length */\n+\tjob->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;\n+\n+\t/* Set IV parameters */\n+\tjob->iv_len_in_bytes = session->iv.length;\n+\n+\t/* Data Parameters */\n+\tjob->src = rte_pktmbuf_mtod(m_src, uint8_t *);\n+\tjob->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);\n+\n+\tswitch (job->hash_alg) {\n+\tcase IMB_AUTH_AES_CCM:\n+\t\tjob->cipher_start_src_offset_in_bytes =\n+\t\t\t\top->sym->aead.data.offset;\n+\t\tjob->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;\n+\t\tjob->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;\n+\t\tjob->msg_len_to_hash_in_bytes = op->sym->aead.data.length;\n+\n+\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\tsession->iv.offset + 1);\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_AES_GMAC:\n+\t\tif (session->cipher.mode == IMB_CIPHER_GCM) {\n+\t\t\tjob->cipher_start_src_offset_in_bytes =\n+\t\t\t\t\top->sym->aead.data.offset;\n+\t\t\tjob->hash_start_src_offset_in_bytes =\n+\t\t\t\t\top->sym->aead.data.offset;\n+\t\t\tjob->msg_len_to_cipher_in_bytes =\n+\t\t\t\t\top->sym->aead.data.length;\n+\t\t\tjob->msg_len_to_hash_in_bytes =\n+\t\t\t\t\top->sym->aead.data.length;\n+\t\t} else {\n+\t\t\tjob->cipher_start_src_offset_in_bytes =\n+\t\t\t\t\top->sym->auth.data.offset;\n+\t\t\tjob->hash_start_src_offset_in_bytes =\n+\t\t\t\t\top->sym->auth.data.offset;\n+\t\t\tjob->msg_len_to_cipher_in_bytes = 0;\n+\t\t\tjob->msg_len_to_hash_in_bytes = 0;\n+\t\t}\n+\n+\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\t\tsession->iv.offset);\n+\t\tbreak;\n+\n+\tcase IMB_AUTH_CHACHA20_POLY1305:\n+\t\tjob->cipher_start_src_offset_in_bytes =\n+\t\t\top->sym->aead.data.offset;\n+\t\tjob->hash_start_src_offset_in_bytes =\n+\t\t\top->sym->aead.data.offset;\n+\t\tjob->msg_len_to_cipher_in_bytes =\n+\t\t\t\top->sym->aead.data.length;\n+\t\tjob->msg_len_to_hash_in_bytes =\n+\t\t\t\t\top->sym->aead.data.length;\n+\n+\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\t\tsession->iv.offset);\n+\t\tbreak;\n+\tdefault:\n+\t\t/* For SNOW3G, length and offsets are already in bits */\n+\t\tjob->cipher_start_src_offset_in_bytes =\n+\t\t\t\top->sym->cipher.data.offset;\n+\t\tjob->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;\n+\n+\t\tjob->hash_start_src_offset_in_bytes = auth_start_offset(op,\n+\t\t\t\tsession, oop);\n+\t\tjob->msg_len_to_hash_in_bytes = op->sym->auth.data.length;\n+\n+\t\tjob->iv = rte_crypto_op_ctod_offset(op, uint8_t *,\n+\t\t\tsession->iv.offset);\n+\t}\n+\n+\tif (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)\n+\t\tjob->msg_len_to_cipher_in_bytes >>= 3;\n+\telse if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)\n+\t\tjob->msg_len_to_hash_in_bytes >>= 3;\n+\n+\t/* Set user data to be crypto operation data struct */\n+\tjob->user_data = op;\n+\n+\treturn 0;\n+}\n+\n+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n+/**\n+ * Process a crypto operation containing a security op and complete a\n+ * IMB_JOB job structure for submission to the multi buffer library for\n+ * processing.\n+ */\n+static inline int\n+set_sec_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,\n+\t\t\tstruct rte_crypto_op *op, uint8_t *digest_idx)\n+{\n+\tstruct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);\n+\tstruct rte_mbuf *m_src, *m_dst;\n+\tstruct rte_crypto_sym_op *sym;\n+\tstruct aesni_mb_session *session = NULL;\n+\n+\tif (unlikely(op->sess_type != RTE_CRYPTO_OP_SECURITY_SESSION)) {\n+\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\t\treturn -1;\n+\t}\n+\tsession = (struct aesni_mb_session *)\n+\t\tget_sec_session_private_data(op->sym->sec_session);\n+\n+\tif (unlikely(session == NULL)) {\n+\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\t\treturn -1;\n+\t}\n+\t/* Only DOCSIS protocol operations supported now */\n+\tif (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||\n+\t\t\tsession->auth.algo != IMB_AUTH_DOCSIS_CRC32) {\n+\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t\treturn -1;\n+\t}\n+\n+\tsym = op->sym;\n+\tm_src = sym->m_src;\n+\n+\tif (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {\n+\t\t/* in-place operation */\n+\t\tm_dst = m_src;\n+\t} else {\n+\t\t/* out-of-place operation not supported */\n+\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* Set crypto operation */\n+\tjob->chain_order = session->chain_order;\n+\n+\t/* Set cipher parameters */\n+\tjob->cipher_direction = session->cipher.direction;\n+\tjob->cipher_mode = session->cipher.mode;\n+\n+\tjob->key_len_in_bytes = session->cipher.key_length_in_bytes;\n+\tjob->enc_keys = session->cipher.expanded_aes_keys.encode;\n+\tjob->dec_keys = session->cipher.expanded_aes_keys.decode;\n+\n+\t/* Set IV parameters */\n+\tjob->iv_len_in_bytes = session->iv.length;\n+\tjob->iv = (uint8_t *)op + session->iv.offset;\n+\n+\t/* Set authentication parameters */\n+\tjob->hash_alg = session->auth.algo;\n+\n+\t/* Set digest output location */\n+\tjob->auth_tag_output = qp_data->temp_digests[*digest_idx];\n+\t*digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;\n+\n+\t/* Set digest length */\n+\tjob->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;\n+\n+\t/* Set data parameters */\n+\tjob->src = rte_pktmbuf_mtod(m_src, uint8_t *);\n+\tjob->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,\n+\t\t\t\t\t\tsym->cipher.data.offset);\n+\n+\tjob->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;\n+\tjob->msg_len_to_cipher_in_bytes = sym->cipher.data.length;\n+\n+\tjob->hash_start_src_offset_in_bytes = sym->auth.data.offset;\n+\tjob->msg_len_to_hash_in_bytes = sym->auth.data.length;\n+\n+\tjob->user_data = op;\n+\n+\treturn 0;\n+}\n+\n+static inline void\n+verify_docsis_sec_crc(IMB_JOB *job, uint8_t *status)\n+{\n+\tuint16_t crc_offset;\n+\tuint8_t *crc;\n+\n+\tif (!job->msg_len_to_hash_in_bytes)\n+\t\treturn;\n+\n+\tcrc_offset = job->hash_start_src_offset_in_bytes +\n+\t\t\tjob->msg_len_to_hash_in_bytes -\n+\t\t\tjob->cipher_start_src_offset_in_bytes;\n+\tcrc = job->dst + crc_offset;\n+\n+\t/* Verify CRC (at the end of the message) */\n+\tif (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0)\n+\t\t*status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n+}\n+#endif\n+\n+static inline void\n+verify_digest(IMB_JOB *job, void *digest, uint16_t len, uint8_t *status)\n+{\n+\t/* Verify digest if required */\n+\tif (memcmp(job->auth_tag_output, digest, len) != 0)\n+\t\t*status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n+}\n+\n+static inline void\n+generate_digest(IMB_JOB *job, struct rte_crypto_op *op,\n+\t\tstruct aesni_mb_session *sess)\n+{\n+\t/* No extra copy needed */\n+\tif (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))\n+\t\treturn;\n+\n+\t/*\n+\t * This can only happen for HMAC, so only digest\n+\t * for authentication algos is required\n+\t */\n+\tmemcpy(op->sym->auth.digest.data, job->auth_tag_output,\n+\t\t\tsess->auth.req_digest_len);\n+}\n+\n+/**\n+ * Process a completed job and return rte_mbuf which job processed\n+ *\n+ * @param qp\tQueue Pair to process\n+ * @param job\tIMB_JOB job to process\n+ *\n+ * @return\n+ * - Returns processed crypto operation.\n+ * - Returns NULL on invalid job\n+ */\n+static inline struct rte_crypto_op *\n+post_process_mb_job(struct ipsec_mb_qp *qp, IMB_JOB *job)\n+{\n+\tstruct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;\n+\tstruct aesni_mb_session *sess = NULL;\n+\tuint32_t driver_id = ipsec_mb_get_driver_id(\n+\t\t\t\t\t\tIPSEC_MB_PMD_TYPE_AESNI_MB);\n+\n+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n+\tuint8_t is_docsis_sec = 0;\n+\n+\tif (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {\n+\t\t/*\n+\t\t * Assuming at this point that if it's a security type op, that\n+\t\t * this is for DOCSIS\n+\t\t */\n+\t\tis_docsis_sec = 1;\n+\t\tsess = get_sec_session_private_data(op->sym->sec_session);\n+\t} else\n+#endif\n+\t{\n+\t\tsess = get_sym_session_private_data(op->sym->session,\n+\t\t\t\t\t\tdriver_id);\n+\t}\n+\n+\tif (unlikely(sess == NULL)) {\n+\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\t\treturn op;\n+\t}\n+\n+\tif (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {\n+\t\tswitch (job->status) {\n+\t\tcase IMB_STATUS_COMPLETED:\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_SUCCESS;\n+\n+\t\t\tif (job->hash_alg == IMB_AUTH_NULL)\n+\t\t\t\tbreak;\n+\n+\t\t\tif (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {\n+\t\t\t\tif (is_aead_algo(job->hash_alg,\n+\t\t\t\t\t\tsess->cipher.mode))\n+\t\t\t\t\tverify_digest(job,\n+\t\t\t\t\t\top->sym->aead.digest.data,\n+\t\t\t\t\t\tsess->auth.req_digest_len,\n+\t\t\t\t\t\t&op->status);\n+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n+\t\t\t\telse if (is_docsis_sec)\n+\t\t\t\t\tverify_docsis_sec_crc(job,\n+\t\t\t\t\t\t&op->status);\n+#endif\n+\t\t\t\telse\n+\t\t\t\t\tverify_digest(job,\n+\t\t\t\t\t\top->sym->auth.digest.data,\n+\t\t\t\t\t\tsess->auth.req_digest_len,\n+\t\t\t\t\t\t&op->status);\n+\t\t\t} else\n+\t\t\t\tgenerate_digest(job, op, sess);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t\t}\n+\t}\n+\n+\t/* Free session if a session-less crypto op */\n+\tif (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {\n+\t\tmemset(sess, 0, sizeof(struct aesni_mb_session));\n+\t\tmemset(op->sym->session, 0,\n+\t\t\trte_cryptodev_sym_get_existing_header_session_size(\n+\t\t\t\top->sym->session));\n+\t\trte_mempool_put(qp->sess_mp_priv, sess);\n+\t\trte_mempool_put(qp->sess_mp, op->sym->session);\n+\t\top->sym->session = NULL;\n+\t}\n+\n+\treturn op;\n+}\n+\n+static inline void\n+post_process_mb_sync_job(IMB_JOB *job)\n+{\n+\tuint32_t *st;\n+\n+\tst = job->user_data;\n+\tst[0] = (job->status == IMB_STATUS_COMPLETED) ? 0 : EBADMSG;\n+}\n+\n+/**\n+ * Process a completed IMB_JOB job and keep processing jobs until\n+ * get_completed_job return NULL\n+ *\n+ * @param qp\t\tQueue Pair to process\n+ * @param mb_mgr\tIMB_MGR to use\n+ * @param job\t\tIMB_JOB job\n+ * @param ops\t\tcrypto ops to fill\n+ * @param nb_ops\tnumber of crypto ops\n+ *\n+ * @return\n+ * - Number of processed jobs\n+ */\n+static unsigned\n+handle_completed_jobs(struct ipsec_mb_qp *qp, IMB_MGR *mb_mgr,\n+\t\tIMB_JOB *job, struct rte_crypto_op **ops,\n+\t\tuint16_t nb_ops)\n+{\n+\tstruct rte_crypto_op *op = NULL;\n+\tuint16_t processed_jobs = 0;\n+\n+\twhile (job != NULL) {\n+\t\top = post_process_mb_job(qp, job);\n+\n+\t\tif (op) {\n+\t\t\tops[processed_jobs++] = op;\n+\t\t\tqp->stats.dequeued_count++;\n+\t\t} else {\n+\t\t\tqp->stats.dequeue_err_count++;\n+\t\t\tbreak;\n+\t\t}\n+\t\tif (processed_jobs == nb_ops)\n+\t\t\tbreak;\n+\n+\t\tjob = IMB_GET_COMPLETED_JOB(mb_mgr);\n+\t}\n+\n+\treturn processed_jobs;\n+}\n+\n+static inline uint32_t\n+handle_completed_sync_jobs(IMB_JOB *job, IMB_MGR *mb_mgr)\n+{\n+\tuint32_t i;\n+\n+\tfor (i = 0; job != NULL; i++, job = IMB_GET_COMPLETED_JOB(mb_mgr))\n+\t\tpost_process_mb_sync_job(job);\n+\n+\treturn i;\n+}\n+\n+static inline uint32_t\n+flush_mb_sync_mgr(IMB_MGR *mb_mgr)\n+{\n+\tIMB_JOB *job;\n+\n+\tjob = IMB_FLUSH_JOB(mb_mgr);\n+\treturn handle_completed_sync_jobs(job, mb_mgr);\n+}\n+\n+static inline uint16_t\n+flush_mb_mgr(struct ipsec_mb_qp *qp, IMB_MGR *mb_mgr,\n+\t\tstruct rte_crypto_op **ops, uint16_t nb_ops)\n+{\n+\tint processed_ops = 0;\n+\n+\t/* Flush the remaining jobs */\n+\tIMB_JOB *job = IMB_FLUSH_JOB(mb_mgr);\n+\n+\tif (job)\n+\t\tprocessed_ops += handle_completed_jobs(qp, mb_mgr, job,\n+\t\t\t\t&ops[processed_ops], nb_ops - processed_ops);\n+\n+\treturn processed_ops;\n+}\n+\n+static inline IMB_JOB *\n+set_job_null_op(IMB_JOB *job, struct rte_crypto_op *op)\n+{\n+\tjob->chain_order = IMB_ORDER_HASH_CIPHER;\n+\tjob->cipher_mode = IMB_CIPHER_NULL;\n+\tjob->hash_alg = IMB_AUTH_NULL;\n+\tjob->cipher_direction = IMB_DIR_DECRYPT;\n+\n+\t/* Set user data to be crypto operation data struct */\n+\tjob->user_data = op;\n+\n+\treturn job;\n+}\n+\n+static uint16_t\n+aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,\n+\t\tuint16_t nb_ops)\n+{\n+\tstruct ipsec_mb_qp *qp = queue_pair;\n+\tIMB_MGR *mb_mgr = qp->mb_mgr;\n+\tstruct rte_crypto_op *op;\n+\tIMB_JOB *job;\n+\tint retval, processed_jobs = 0;\n+\n+\tif (unlikely(nb_ops == 0 || mb_mgr == NULL))\n+\t\treturn 0;\n+\n+\tuint8_t digest_idx = qp->digest_idx;\n+\n+\tdo {\n+\t\t/* Get next free mb job struct from mb manager */\n+\t\tjob = IMB_GET_NEXT_JOB(mb_mgr);\n+\t\tif (unlikely(job == NULL)) {\n+\t\t\t/* if no free mb job structs we need to flush mb_mgr */\n+\t\t\tprocessed_jobs += flush_mb_mgr(qp, mb_mgr,\n+\t\t\t\t\t&ops[processed_jobs],\n+\t\t\t\t\tnb_ops - processed_jobs);\n+\n+\t\t\tif (nb_ops == processed_jobs)\n+\t\t\t\tbreak;\n+\n+\t\t\tjob = IMB_GET_NEXT_JOB(mb_mgr);\n+\t\t}\n+\n+\t\t/*\n+\t\t * Get next operation to process from ingress queue.\n+\t\t * There is no need to return the job to the IMB_MGR\n+\t\t * if there are no more operations to process, since the IMB_MGR\n+\t\t * can use that pointer again in next get_next calls.\n+\t\t */\n+\t\tretval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);\n+\t\tif (retval < 0)\n+\t\t\tbreak;\n+\n+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n+\t\tif (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)\n+\t\t\tretval = set_sec_mb_job_params(job, qp, op,\n+\t\t\t\t\t\t&digest_idx);\n+\t\telse\n+#endif\n+\t\t\tretval = set_mb_job_params(job, qp, op,\n+\t\t\t\t&digest_idx);\n+\n+\t\tif (unlikely(retval != 0)) {\n+\t\t\tqp->stats.dequeue_err_count++;\n+\t\t\tset_job_null_op(job, op);\n+\t\t}\n+\n+\t\t/* Submit job to multi-buffer for processing */\n+#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG\n+\t\tjob = IMB_SUBMIT_JOB(mb_mgr);\n+#else\n+\t\tjob = IMB_SUBMIT_JOB_NOCHECK(mb_mgr);\n+#endif\n+\t\t/*\n+\t\t * If submit returns a processed job then handle it,\n+\t\t * before submitting subsequent jobs\n+\t\t */\n+\t\tif (job)\n+\t\t\tprocessed_jobs += handle_completed_jobs(qp, mb_mgr,\n+\t\t\t\t\tjob, &ops[processed_jobs],\n+\t\t\t\t\tnb_ops - processed_jobs);\n+\n+\t} while (processed_jobs < nb_ops);\n+\n+\tqp->digest_idx = digest_idx;\n+\n+\tif (processed_jobs < 1)\n+\t\tprocessed_jobs += flush_mb_mgr(qp, mb_mgr,\n+\t\t\t\t&ops[processed_jobs],\n+\t\t\t\tnb_ops - processed_jobs);\n+\n+\treturn processed_jobs;\n+}\n+\n+\n+static inline void\n+ipsec_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)\n+{\n+\tuint32_t i;\n+\n+\tfor (i = 0; i != vec->num; ++i)\n+\t\tvec->status[i] = err;\n+}\n+\n+static inline int\n+check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)\n+{\n+\t/* no multi-seg support with current AESNI-MB PMD */\n+\tif (sgl->num != 1)\n+\t\treturn -ENOTSUP;\n+\telse if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)\n+\t\treturn -EINVAL;\n+\treturn 0;\n+}\n+\n+static inline IMB_JOB *\n+submit_sync_job(IMB_MGR *mb_mgr)\n+{\n+#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG\n+\treturn IMB_SUBMIT_JOB(mb_mgr);\n+#else\n+\treturn IMB_SUBMIT_JOB_NOCHECK(mb_mgr);\n+#endif\n+}\n+\n+static inline uint32_t\n+generate_sync_dgst(struct rte_crypto_sym_vec *vec,\n+\tconst uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)\n+{\n+\tuint32_t i, k;\n+\n+\tfor (i = 0, k = 0; i != vec->num; i++) {\n+\t\tif (vec->status[i] == 0) {\n+\t\t\tmemcpy(vec->digest[i].va, dgst[i], len);\n+\t\t\tk++;\n+\t\t}\n+\t}\n+\n+\treturn k;\n+}\n+\n+static inline uint32_t\n+verify_sync_dgst(struct rte_crypto_sym_vec *vec,\n+\tconst uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)\n+{\n+\tuint32_t i, k;\n+\n+\tfor (i = 0, k = 0; i != vec->num; i++) {\n+\t\tif (vec->status[i] == 0) {\n+\t\t\tif (memcmp(vec->digest[i].va, dgst[i], len) != 0)\n+\t\t\t\tvec->status[i] = EBADMSG;\n+\t\t\telse\n+\t\t\t\tk++;\n+\t\t}\n+\t}\n+\n+\treturn k;\n+}\n+\n+static uint32_t\n+aesni_mb_process_bulk(struct rte_cryptodev *dev,\n+\tstruct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,\n+\tstruct rte_crypto_sym_vec *vec)\n+{\n+\tint32_t ret;\n+\tuint32_t i, j, k, len;\n+\tvoid *buf;\n+\tIMB_JOB *job;\n+\tIMB_MGR *mb_mgr;\n+\tstruct aesni_mb_session *s;\n+\tuint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];\n+\n+\ts = get_sym_session_private_data(sess, dev->driver_id);\n+\tif (s == NULL) {\n+\t\tipsec_mb_fill_error_code(vec, EINVAL);\n+\t\treturn 0;\n+\t}\n+\n+\t/* get per-thread MB MGR, create one if needed */\n+\tmb_mgr = get_per_thread_mb_mgr();\n+\tif (unlikely(mb_mgr == NULL))\n+\t\treturn 0;\n+\n+\tfor (i = 0, j = 0, k = 0; i != vec->num; i++) {\n+\t\tret = check_crypto_sgl(sofs, vec->sgl + i);\n+\t\tif (ret != 0) {\n+\t\t\tvec->status[i] = ret;\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tbuf = vec->sgl[i].vec[0].base;\n+\t\tlen = vec->sgl[i].vec[0].len;\n+\n+\t\tjob = IMB_GET_NEXT_JOB(mb_mgr);\n+\t\tif (job == NULL) {\n+\t\t\tk += flush_mb_sync_mgr(mb_mgr);\n+\t\t\tjob = IMB_GET_NEXT_JOB(mb_mgr);\n+\t\t\tRTE_ASSERT(job != NULL);\n+\t\t}\n+\n+\t\t/* Submit job for processing */\n+\t\tset_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i],\n+\t\t\t&vec->aad[i], tmp_dgst[i], &vec->status[i]);\n+\t\tjob = submit_sync_job(mb_mgr);\n+\t\tj++;\n+\n+\t\t/* handle completed jobs */\n+\t\tk += handle_completed_sync_jobs(job, mb_mgr);\n+\t}\n+\n+\t/* flush remaining jobs */\n+\twhile (k != j)\n+\t\tk += flush_mb_sync_mgr(mb_mgr);\n+\n+\t/* finish processing for successful jobs: check/update digest */\n+\tif (k != 0) {\n+\t\tif (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)\n+\t\t\tk = verify_sync_dgst(vec,\n+\t\t\t\t(const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,\n+\t\t\t\ts->auth.req_digest_len);\n+\t\telse\n+\t\t\tk = generate_sync_dgst(vec,\n+\t\t\t\t(const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,\n+\t\t\t\ts->auth.req_digest_len);\n+\t}\n+\n+\treturn k;\n+}\n+\n+struct rte_cryptodev_ops aes_mb_pmd_ops = {\n+\t.dev_configure = ipsec_mb_pmd_config,\n+\t.dev_start = ipsec_mb_pmd_start,\n+\t.dev_stop = ipsec_mb_pmd_stop,\n+\t.dev_close = ipsec_mb_pmd_close,\n+\n+\t.stats_get = ipsec_mb_pmd_stats_get,\n+\t.stats_reset = ipsec_mb_pmd_stats_reset,\n+\n+\t.dev_infos_get = ipsec_mb_pmd_info_get,\n+\n+\t.queue_pair_setup = ipsec_mb_pmd_qp_setup,\n+\t.queue_pair_release = ipsec_mb_pmd_qp_release,\n+\n+\t.sym_cpu_process = aesni_mb_process_bulk,\n+\n+\t.sym_session_get_size = ipsec_mb_pmd_sym_session_get_size,\n+\t.sym_session_configure = ipsec_mb_pmd_sym_session_configure,\n+\t.sym_session_clear = ipsec_mb_pmd_sym_session_clear\n+};\n+\n+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n+/**\n+ * Configure a aesni multi-buffer session from a security session\n+ * configuration\n+ */\n+static int\n+aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,\n+\t\tstruct rte_security_session *sess,\n+\t\tstruct rte_mempool *mempool)\n+{\n+\tvoid *sess_private_data;\n+\tstruct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;\n+\tint ret;\n+\n+\tif (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||\n+\t\t\tconf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {\n+\t\tIPSEC_MB_LOG(ERR, \"Invalid security protocol\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (rte_mempool_get(mempool, &sess_private_data)) {\n+\t\tIPSEC_MB_LOG(ERR, \"Couldn't get object from session mempool\");\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,\n+\t\t\tsess_private_data);\n+\n+\tif (ret != 0) {\n+\t\tIPSEC_MB_LOG(ERR, \"Failed to configure session parameters\");\n+\n+\t\t/* Return session to mempool */\n+\t\trte_mempool_put(mempool, sess_private_data);\n+\t\treturn ret;\n+\t}\n+\n+\tset_sec_session_private_data(sess, sess_private_data);\n+\n+\treturn ret;\n+}\n+\n+/** Clear the memory of session so it does not leave key material behind */\n+static int\n+aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,\n+\t\tstruct rte_security_session *sess)\n+{\n+\tvoid *sess_priv = get_sec_session_private_data(sess);\n+\n+\tif (sess_priv) {\n+\t\tstruct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);\n+\n+\t\tmemset(sess_priv, 0, sizeof(struct aesni_mb_session));\n+\t\tset_sec_session_private_data(sess, NULL);\n+\t\trte_mempool_put(sess_mp, sess_priv);\n+\t}\n+\treturn 0;\n+}\n+\n+static const struct rte_cryptodev_capabilities\n+\t\t\t\t\taesni_mb_pmd_security_crypto_cap[] = {\n+\t{\t/* AES DOCSIS BPI */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,\n+\t\t\t{.cipher = {\n+\t\t\t\t.algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 16\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\n+\tRTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()\n+};\n+\n+static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {\n+\t{\t/* DOCSIS Uplink */\n+\t\t.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,\n+\t\t.protocol = RTE_SECURITY_PROTOCOL_DOCSIS,\n+\t\t.docsis = {\n+\t\t\t.direction = RTE_SECURITY_DOCSIS_UPLINK\n+\t\t},\n+\t\t.crypto_capabilities = aesni_mb_pmd_security_crypto_cap\n+\t},\n+\t{\t/* DOCSIS Downlink */\n+\t\t.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,\n+\t\t.protocol = RTE_SECURITY_PROTOCOL_DOCSIS,\n+\t\t.docsis = {\n+\t\t\t.direction = RTE_SECURITY_DOCSIS_DOWNLINK\n+\t\t},\n+\t\t.crypto_capabilities = aesni_mb_pmd_security_crypto_cap\n+\t},\n+\t{\n+\t\t.action = RTE_SECURITY_ACTION_TYPE_NONE\n+\t}\n+};\n+\n+/** Get security capabilities for aesni multi-buffer */\n+static const struct rte_security_capability *\n+aesni_mb_pmd_sec_capa_get(void *device __rte_unused)\n+{\n+\treturn aesni_mb_pmd_security_cap;\n+}\n+\n+static struct rte_security_ops aesni_mb_pmd_sec_ops = {\n+\t\t.session_create = aesni_mb_pmd_sec_sess_create,\n+\t\t.session_update = NULL,\n+\t\t.session_stats_get = NULL,\n+\t\t.session_destroy = aesni_mb_pmd_sec_sess_destroy,\n+\t\t.set_pkt_metadata = NULL,\n+\t\t.capabilities_get = aesni_mb_pmd_sec_capa_get\n+};\n+\n+struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;\n+\n+static int\n+aesni_mb_configure_dev(struct rte_cryptodev *dev)\n+{\n+\tstruct rte_security_ctx *security_instance;\n+\n+\tsecurity_instance = rte_malloc(\"aesni_mb_sec\",\n+\t\t\t\tsizeof(struct rte_security_ctx),\n+\t\t\t\tRTE_CACHE_LINE_SIZE);\n+\tif (security_instance != NULL) {\n+\t\tsecurity_instance->device = (void *)dev;\n+\t\tsecurity_instance->ops = rte_aesni_mb_pmd_sec_ops;\n+\t\tsecurity_instance->sess_cnt = 0;\n+\t\tdev->security_ctx = security_instance;\n+\n+\t\treturn 0;\n+\t}\n+\n+\treturn -ENOMEM;\n+}\n+\n+#endif\n+\n+static int\n+cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)\n+{\n+\treturn cryptodev_ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_AESNI_MB);\n+}\n+\n+static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {\n+\t.probe = cryptodev_aesni_mb_probe,\n+\t.remove = cryptodev_ipsec_mb_remove\n+};\n+\n+static struct cryptodev_driver aesni_mb_crypto_drv;\n+\n+RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD,\n+\tcryptodev_aesni_mb_pmd_drv);\n+RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);\n+RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,\n+\t\t\t\"max_nb_queue_pairs=<int> socket_id=<int>\");\n+RTE_PMD_REGISTER_CRYPTO_DRIVER(\n+\taesni_mb_crypto_drv,\n+\tcryptodev_aesni_mb_pmd_drv.driver,\n+\tpmd_driver_id_aesni_mb);\n+\n+/* Constructor function to register aesni-mb PMD */\n+RTE_INIT(ipsec_mb_register_aesni_mb)\n+{\n+\tstruct ipsec_mb_pmd_data *aesni_mb_data =\n+\t\t&ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_AESNI_MB];\n+\n+\taesni_mb_data->caps = aesni_mb_capabilities;\n+\taesni_mb_data->dequeue_burst = aesni_mb_dequeue_burst;\n+\taesni_mb_data->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |\n+\t\t\tRTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |\n+\t\t\tRTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |\n+\t\t\tRTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |\n+\t\t\tRTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |\n+\t\t\tRTE_CRYPTODEV_FF_SYM_SESSIONLESS;\n+\n+\taesni_mb_data->internals_priv_size = 0;\n+\taesni_mb_data->ops = &aes_mb_pmd_ops;\n+\taesni_mb_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);\n+\taesni_mb_data->queue_pair_configure = NULL;\n+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n+\taesni_mb_data->security_ops = &aesni_mb_pmd_sec_ops;\n+\taesni_mb_data->dev_config = aesni_mb_configure_dev;\n+\taesni_mb_data->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;\n+#endif\n+\taesni_mb_data->session_configure = aesni_mb_session_configure;\n+\taesni_mb_data->session_priv_size = sizeof(struct aesni_mb_session);\n+}\ndiff --git a/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c b/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c\nindex 3f2cefed52..e71037f345 100644\n--- a/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c\n+++ b/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c\n@@ -105,6 +105,7 @@ cryptodev_ipsec_mb_create(struct rte_vdev_device *vdev,\n \tdev->dev_ops = ipsec_mb_pmds[pmd_type].ops;\n \tdev->enqueue_burst = ipsec_mb_pmd_enqueue_burst;\n \tdev->dequeue_burst = ipsec_mb_pmds[pmd_type].dequeue_burst;\n+\tdev->feature_flags = pmd_data->feature_flags;\n \n \tif (pmd_data->dev_config) {\n \t\tretval = (*pmd_data->dev_config)(dev);\n@@ -116,8 +117,6 @@ cryptodev_ipsec_mb_create(struct rte_vdev_device *vdev,\n \t\t}\n \t}\n \n-\tdev->feature_flags = pmd_data->feature_flags;\n-\n \tswitch (vector_mode) {\n \tcase IPSEC_MB_AVX512:\n \t\tdev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;\n@@ -164,6 +163,10 @@ cryptodev_ipsec_mb_remove(struct rte_vdev_device *vdev)\n \t\trte_free(cryptodev->security_ctx);\n \t\tcryptodev->security_ctx = NULL;\n \t}\n+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED\n+\trte_free(cryptodev->security_ctx);\n+\tcryptodev->security_ctx = NULL;\n+#endif\n \n \treturn rte_cryptodev_pmd_destroy(cryptodev);\n }\ndiff --git a/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd_private.h b/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd_private.h\nindex 35860b1b10..2b589eee47 100644\n--- a/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd_private.h\n+++ b/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd_private.h\n@@ -34,6 +34,9 @@ extern enum ipsec_mb_vector_mode vector_mode;\n /** IMB_MGR instances, one per thread */\n extern RTE_DEFINE_PER_LCORE(IMB_MGR *, mb_mgr);\n \n+#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb\n+/**< IPSEC Multi buffer aesni_mb PMD device name */\n+\n /** PMD LOGTYPE DRIVER, common to all PMDs */\n extern int ipsec_mb_logtype_driver;\n #define IPSEC_MB_LOG(level, fmt, ...)                                         \\\n@@ -42,6 +45,7 @@ extern int ipsec_mb_logtype_driver;\n \n /** All supported device types */\n enum ipsec_mb_pmd_types {\n+\tIPSEC_MB_PMD_TYPE_AESNI_MB = 0,\n \tIPSEC_MB_N_PMD_TYPES\n };\n \n@@ -60,10 +64,18 @@ enum ipsec_mb_operation {\n \tIPSEC_MB_OP_NOT_SUPPORTED\n };\n \n+extern uint8_t pmd_driver_id_aesni_mb;\n+\n /** Helper function. Gets driver ID based on PMD type */\n static __rte_always_inline uint8_t\n-ipsec_mb_get_driver_id(__rte_unused enum ipsec_mb_pmd_types pmd_type)\n+ipsec_mb_get_driver_id(enum ipsec_mb_pmd_types pmd_type)\n {\n+\tswitch (pmd_type) {\n+\tcase IPSEC_MB_PMD_TYPE_AESNI_MB:\n+\t\treturn pmd_driver_id_aesni_mb;\n+\tdefault:\n+\t\tbreak;\n+\t}\n \treturn UINT8_MAX;\n }\n \n@@ -136,6 +148,135 @@ get_per_thread_mb_mgr(void)\n \treturn RTE_PER_LCORE(mb_mgr);\n }\n \n+/** Helper function. Gets mode and chained xforms from the xform */\n+static __rte_always_inline int\n+ipsec_mb_parse_xform(const struct rte_crypto_sym_xform *xform,\n+\t\t\tenum ipsec_mb_operation *mode,\n+\t\t\tconst struct rte_crypto_sym_xform **auth_xform,\n+\t\t\tconst struct rte_crypto_sym_xform **cipher_xform,\n+\t\t\tconst struct rte_crypto_sym_xform **aead_xform)\n+{\n+\tconst struct rte_crypto_sym_xform *next = xform->next;\n+\n+\tif (xform == NULL) {\n+\t\t*mode = IPSEC_MB_OP_NOT_SUPPORTED;\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\tif (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {\n+\t\tif (next == NULL) {\n+\t\t\tif (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {\n+\t\t\t\t*mode = IPSEC_MB_OP_ENCRYPT_ONLY;\n+\t\t\t\t*cipher_xform = xform;\n+\t\t\t\t*auth_xform = NULL;\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\t*mode = IPSEC_MB_OP_DECRYPT_ONLY;\n+\t\t\t*cipher_xform = xform;\n+\t\t\t*auth_xform = NULL;\n+\t\t\treturn 0;\n+\t\t}\n+\n+\t\tif (next->type != RTE_CRYPTO_SYM_XFORM_AUTH) {\n+\t\t\t*mode = IPSEC_MB_OP_NOT_SUPPORTED;\n+\t\t\treturn -ENOTSUP;\n+\t\t}\n+\n+\t\tif (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {\n+\t\t\tif (next->auth.op != RTE_CRYPTO_AUTH_OP_GENERATE) {\n+\t\t\t\t*mode = IPSEC_MB_OP_NOT_SUPPORTED;\n+\t\t\t\treturn -ENOTSUP;\n+\t\t\t}\n+\n+\t\t\t*mode = IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN;\n+\t\t\t*cipher_xform = xform;\n+\t\t\t*auth_xform = xform->next;\n+\t\t\treturn 0;\n+\t\t}\n+\t\tif (next->auth.op != RTE_CRYPTO_AUTH_OP_VERIFY) {\n+\t\t\t*mode = IPSEC_MB_OP_NOT_SUPPORTED;\n+\t\t\treturn -ENOTSUP;\n+\t\t}\n+\n+\t\t*mode = IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY;\n+\t\t*cipher_xform = xform;\n+\t\t*auth_xform = xform->next;\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {\n+\t\tif (next == NULL) {\n+\t\t\tif (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {\n+\t\t\t\t*mode = IPSEC_MB_OP_HASH_GEN_ONLY;\n+\t\t\t\t*auth_xform = xform;\n+\t\t\t\t*cipher_xform = NULL;\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\t*mode = IPSEC_MB_OP_HASH_VERIFY_ONLY;\n+\t\t\t*auth_xform = xform;\n+\t\t\t*cipher_xform = NULL;\n+\t\t\treturn 0;\n+\t\t}\n+\n+\t\tif (next->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {\n+\t\t\t*mode = IPSEC_MB_OP_NOT_SUPPORTED;\n+\t\t\treturn -ENOTSUP;\n+\t\t}\n+\n+\t\tif (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {\n+\t\t\tif (next->cipher.op != RTE_CRYPTO_CIPHER_OP_ENCRYPT) {\n+\t\t\t\t*mode = IPSEC_MB_OP_NOT_SUPPORTED;\n+\t\t\t\treturn -ENOTSUP;\n+\t\t\t}\n+\n+\t\t\t*mode = IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT;\n+\t\t\t*auth_xform = xform;\n+\t\t\t*cipher_xform = xform->next;\n+\t\t\treturn 0;\n+\t\t}\n+\t\tif (next->cipher.op != RTE_CRYPTO_CIPHER_OP_DECRYPT) {\n+\t\t\t*mode = IPSEC_MB_OP_NOT_SUPPORTED;\n+\t\t\treturn -ENOTSUP;\n+\t\t}\n+\n+\t\t*mode = IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT;\n+\t\t*auth_xform = xform;\n+\t\t*cipher_xform = xform->next;\n+\t\treturn 0;\n+\t}\n+\n+\tif (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {\n+\t\tif (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {\n+\t\t\t/*\n+\t\t\t * CCM requires to hash first and cipher later\n+\t\t\t * when encrypting\n+\t\t\t */\n+\t\t\tif (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {\n+\t\t\t\t*mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT;\n+\t\t\t\t*aead_xform = xform;\n+\t\t\t\treturn 0;\n+\t\t\t\t} else {\n+\t\t\t\t\t*mode =\n+\t\t\t\tIPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT;\n+\t\t\t\t\t*aead_xform = xform;\n+\t\t\t\t\treturn 0;\n+\t\t\t\t}\n+\t\t} else {\n+\t\t\tif (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {\n+\t\t\t\t*mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT;\n+\t\t\t\t*aead_xform = xform;\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\t*mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT;\n+\t\t\t*aead_xform = xform;\n+\t\t\treturn 0;\n+\t\t}\n+\t}\n+\n+\t*mode = IPSEC_MB_OP_NOT_SUPPORTED;\n+\treturn -ENOTSUP;\n+}\n+\n /** Device creation function */\n int\n cryptodev_ipsec_mb_create(struct rte_vdev_device *vdev,\ndiff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build\nindex e40b18b17b..b2ccea6f94 100644\n--- a/drivers/crypto/meson.build\n+++ b/drivers/crypto/meson.build\n@@ -8,7 +8,6 @@ endif\n drivers = [\n         'ipsec_mb',\n         'aesni_gcm',\n-        'aesni_mb',\n         'armv8',\n         'bcmfs',\n         'caam_jr',\n",
    "prefixes": [
        "v3",
        "03/10"
    ]
}