@@ -1051,13 +1051,6 @@ F: drivers/crypto/aesni_gcm/
F: doc/guides/cryptodevs/aesni_gcm.rst
F: doc/guides/cryptodevs/features/aesni_gcm.ini
-Intel AES-NI Multi-Buffer
-M: Declan Doherty <declan.doherty@intel.com>
-M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
-F: drivers/crypto/aesni_mb/
-F: doc/guides/cryptodevs/aesni_mb.rst
-F: doc/guides/cryptodevs/features/aesni_mb.ini
-
Intel QuickAssist
M: John Griffin <john.griffin@intel.com>
M: Fiona Trahe <fiona.trahe@intel.com>
@@ -1069,7 +1062,10 @@ F: doc/guides/cryptodevs/features/qat.ini
IPsec MB
M: Fan Zhang <roy.fan.zhang@intel.com>
+M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
F: drivers/crypto/ipsec_mb/
+F: doc/guides/cryptodevs/aesni_mb.rst
+F: doc/guides/cryptodevs/features/aesni_mb.ini
KASUMI
M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
@@ -130,7 +130,9 @@ and the Multi-Buffer library version supported by them:
18.02 0.48
18.05 - 19.02 0.49 - 0.52
19.05 - 19.08 0.52
- 19.11+ 0.52 - 1.0*
+ 19.11 - 20.08 0.52 - 0.55
+ 20.11 - 21.08 0.53 - 1.0*
+ 21.11+ 1.0*
============== ============================
\* Multi-buffer library 1.0 or newer only works for Meson but not Make build system.
@@ -101,6 +101,10 @@ New Features
* Multiprocess support was added for the consolidated PMDs,
which requires v1.1 of the intel-ipsec-mb library.
+ * The following PMDs were moved into a single source folder,
+ however their usage and EAL options remain unchanged.
+
+ * AESNI_MB PMD.
* **Updated Marvell cnxk ethdev driver.**
deleted file mode 100644
@@ -1,337 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2016 Intel Corporation
- */
-
-#ifndef _AESNI_MB_PMD_PRIVATE_H_
-#define _AESNI_MB_PMD_PRIVATE_H_
-
-#include <intel-ipsec-mb.h>
-
-#if defined(RTE_LIB_SECURITY) && (IMB_VERSION_NUM) >= IMB_VERSION(0, 54, 0)
-#define AESNI_MB_DOCSIS_SEC_ENABLED 1
-#include <rte_security.h>
-#include <rte_security_driver.h>
-#endif
-
-enum aesni_mb_vector_mode {
- RTE_AESNI_MB_NOT_SUPPORTED = 0,
- RTE_AESNI_MB_SSE,
- RTE_AESNI_MB_AVX,
- RTE_AESNI_MB_AVX2,
- RTE_AESNI_MB_AVX512
-};
-
-#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb
-/**< AES-NI Multi buffer PMD device name */
-
-/** AESNI_MB PMD LOGTYPE DRIVER */
-extern int aesni_mb_logtype_driver;
-
-#define AESNI_MB_LOG(level, fmt, ...) \
- rte_log(RTE_LOG_ ## level, aesni_mb_logtype_driver, \
- "%s() line %u: " fmt "\n", __func__, __LINE__, \
- ## __VA_ARGS__)
-
-
-#define HMAC_IPAD_VALUE (0x36)
-#define HMAC_OPAD_VALUE (0x5C)
-
-/* Maximum length for digest */
-#define DIGEST_LENGTH_MAX 64
-static const unsigned auth_blocksize[] = {
- [NULL_HASH] = 0,
- [MD5] = 64,
- [SHA1] = 64,
- [SHA_224] = 64,
- [SHA_256] = 64,
- [SHA_384] = 128,
- [SHA_512] = 128,
- [AES_XCBC] = 16,
- [AES_CCM] = 16,
- [AES_CMAC] = 16,
- [AES_GMAC] = 16,
- [PLAIN_SHA1] = 64,
- [PLAIN_SHA_224] = 64,
- [PLAIN_SHA_256] = 64,
- [PLAIN_SHA_384] = 128,
- [PLAIN_SHA_512] = 128,
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- [IMB_AUTH_ZUC_EIA3_BITLEN] = 16,
- [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 16,
- [IMB_AUTH_KASUMI_UIA1] = 16
-#endif
-};
-
-/**
- * Get the blocksize in bytes for a specified authentication algorithm
- *
- * @Note: this function will not return a valid value for a non-valid
- * authentication algorithm
- */
-static inline unsigned
-get_auth_algo_blocksize(JOB_HASH_ALG algo)
-{
- return auth_blocksize[algo];
-}
-
-static const unsigned auth_truncated_digest_byte_lengths[] = {
- [MD5] = 12,
- [SHA1] = 12,
- [SHA_224] = 14,
- [SHA_256] = 16,
- [SHA_384] = 24,
- [SHA_512] = 32,
- [AES_XCBC] = 12,
- [AES_CMAC] = 12,
- [AES_CCM] = 8,
- [NULL_HASH] = 0,
- [AES_GMAC] = 12,
- [PLAIN_SHA1] = 20,
- [PLAIN_SHA_224] = 28,
- [PLAIN_SHA_256] = 32,
- [PLAIN_SHA_384] = 48,
- [PLAIN_SHA_512] = 64,
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
- [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
- [IMB_AUTH_KASUMI_UIA1] = 4
-#endif
-};
-
-/**
- * Get the IPsec specified truncated length in bytes of the HMAC digest for a
- * specified authentication algorithm
- *
- * @Note: this function will not return a valid value for a non-valid
- * authentication algorithm
- */
-static inline unsigned
-get_truncated_digest_byte_length(JOB_HASH_ALG algo)
-{
- return auth_truncated_digest_byte_lengths[algo];
-}
-
-static const unsigned auth_digest_byte_lengths[] = {
- [MD5] = 16,
- [SHA1] = 20,
- [SHA_224] = 28,
- [SHA_256] = 32,
- [SHA_384] = 48,
- [SHA_512] = 64,
- [AES_XCBC] = 16,
- [AES_CMAC] = 16,
- [AES_CCM] = 16,
- [AES_GMAC] = 16,
- [NULL_HASH] = 0,
- [PLAIN_SHA1] = 20,
- [PLAIN_SHA_224] = 28,
- [PLAIN_SHA_256] = 32,
- [PLAIN_SHA_384] = 48,
- [PLAIN_SHA_512] = 64,
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
- [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
- [IMB_AUTH_KASUMI_UIA1] = 4
-#endif
- /**< Vector mode dependent pointer table of the multi-buffer APIs */
-
-};
-
-/**
- * Get the full digest size in bytes for a specified authentication algorithm
- * (if available in the Multi-buffer library)
- *
- * @Note: this function will not return a valid value for a non-valid
- * authentication algorithm
- */
-static inline unsigned
-get_digest_byte_length(JOB_HASH_ALG algo)
-{
- return auth_digest_byte_lengths[algo];
-}
-
-enum aesni_mb_operation {
- AESNI_MB_OP_HASH_CIPHER,
- AESNI_MB_OP_CIPHER_HASH,
- AESNI_MB_OP_HASH_ONLY,
- AESNI_MB_OP_CIPHER_ONLY,
- AESNI_MB_OP_AEAD_HASH_CIPHER,
- AESNI_MB_OP_AEAD_CIPHER_HASH,
- AESNI_MB_OP_NOT_SUPPORTED
-};
-
-/** private data structure for each virtual AESNI device */
-struct aesni_mb_private {
- enum aesni_mb_vector_mode vector_mode;
- /**< CPU vector instruction set mode */
- unsigned max_nb_queue_pairs;
- /**< Max number of queue pairs supported by device */
- MB_MGR *mb_mgr;
- /**< Multi-buffer instance */
-};
-
-/** AESNI Multi buffer queue pair */
-struct aesni_mb_qp {
- uint16_t id;
- /**< Queue Pair Identifier */
- char name[RTE_CRYPTODEV_NAME_MAX_LEN];
- /**< Unique Queue Pair Name */
- MB_MGR *mb_mgr;
- /**< Multi-buffer instance */
- struct rte_ring *ingress_queue;
- /**< Ring for placing operations ready for processing */
- struct rte_mempool *sess_mp;
- /**< Session Mempool */
- struct rte_mempool *sess_mp_priv;
- /**< Session Private Data Mempool */
- struct rte_cryptodev_stats stats;
- /**< Queue pair statistics */
- uint8_t digest_idx;
- /**< Index of the next slot to be used in temp_digests,
- * to store the digest for a given operation
- */
- uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];
- /**< Buffers used to store the digest generated
- * by the driver when verifying a digest provided
- * by the user (using authentication verify operation)
- */
-} __rte_cache_aligned;
-
-/** AES-NI multi-buffer private session structure */
-struct aesni_mb_session {
- JOB_CHAIN_ORDER chain_order;
- struct {
- uint16_t length;
- uint16_t offset;
- } iv;
- struct {
- uint16_t length;
- uint16_t offset;
- } auth_iv;
- /**< IV parameters */
-
- /** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;
- /**< Vector mode dependent pointer table of the multi-buffer APIs */
-
- struct {
- /** Cipher direction - encrypt / decrypt */
- JOB_CIPHER_DIRECTION direction;
- /** Cipher mode - CBC / Counter */
- JOB_CIPHER_MODE mode;
-
- uint64_t key_length_in_bytes;
-
- union {
- struct {
- uint32_t encode[60] __rte_aligned(16);
- /**< encode key */
- uint32_t decode[60] __rte_aligned(16);
- /**< decode key */
- } expanded_aes_keys;
- /**< Expanded AES keys - Allocating space to
- * contain the maximum expanded key size which
- * is 240 bytes for 256 bit AES, calculate by:
- * ((key size (bytes)) *
- * ((number of rounds) + 1))
- */
- struct {
- const void *ks_ptr[3];
- uint64_t key[3][16];
- } exp_3des_keys;
- /**< Expanded 3DES keys */
-
- struct gcm_key_data gcm_key;
- /**< Expanded GCM key */
- uint8_t zuc_cipher_key[16];
- /**< ZUC cipher key */
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- snow3g_key_schedule_t pKeySched_snow3g_cipher;
- /**< SNOW3G scheduled cipher key */
- kasumi_key_sched_t pKeySched_kasumi_cipher;
- /**< KASUMI scheduled cipher key */
-#endif
- };
- } cipher;
-
- /** Authentication Parameters */
- struct {
- JOB_HASH_ALG algo; /**< Authentication Algorithm */
- enum rte_crypto_auth_operation operation;
- /**< auth operation generate or verify */
- union {
- struct {
- uint8_t inner[128] __rte_aligned(16);
- /**< inner pad */
- uint8_t outer[128] __rte_aligned(16);
- /**< outer pad */
- } pads;
- /**< HMAC Authentication pads -
- * allocating space for the maximum pad
- * size supported which is 128 bytes for
- * SHA512
- */
-
- struct {
- uint32_t k1_expanded[44] __rte_aligned(16);
- /**< k1 (expanded key). */
- uint8_t k2[16] __rte_aligned(16);
- /**< k2. */
- uint8_t k3[16] __rte_aligned(16);
- /**< k3. */
- } xcbc;
-
- struct {
- uint32_t expkey[60] __rte_aligned(16);
- /**< k1 (expanded key). */
- uint32_t skey1[4] __rte_aligned(16);
- /**< k2. */
- uint32_t skey2[4] __rte_aligned(16);
- /**< k3. */
- } cmac;
- /**< Expanded XCBC authentication keys */
- uint8_t zuc_auth_key[16];
- /**< ZUC authentication key */
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- snow3g_key_schedule_t pKeySched_snow3g_auth;
- /**< SNOW3G scheduled authentication key */
- kasumi_key_sched_t pKeySched_kasumi_auth;
- /**< KASUMI scheduled authentication key */
-#endif
- };
- /** Generated digest size by the Multi-buffer library */
- uint16_t gen_digest_len;
- /** Requested digest size from Cryptodev */
- uint16_t req_digest_len;
-
- } auth;
- struct {
- /** AAD data length */
- uint16_t aad_len;
- } aead;
-} __rte_cache_aligned;
-
-extern int
-aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
- struct aesni_mb_session *sess,
- const struct rte_crypto_sym_xform *xform);
-
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
-extern int
-aesni_mb_set_docsis_sec_session_parameters(
- __rte_unused struct rte_cryptodev *dev,
- struct rte_security_session_conf *conf,
- void *sess);
-#endif
-
-/** device specific operations function pointer structures */
-extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
-extern struct rte_security_ops *rte_aesni_mb_pmd_sec_ops;
-#endif
-
-extern uint32_t
-aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
- struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
- struct rte_crypto_sym_vec *vec);
-
-#endif /* _AESNI_MB_PMD_PRIVATE_H_ */
deleted file mode 100644
@@ -1,25 +0,0 @@
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2018 Intel Corporation
-
-IMB_required_ver = '0.52.0'
-lib = cc.find_library('IPSec_MB', required: false)
-if not lib.found()
- build = false
- reason = 'missing dependency, "libIPSec_MB"'
-else
- ext_deps += lib
-
- # version comes with quotes, so we split based on " and take the middle
- imb_ver = cc.get_define('IMB_VERSION_STR',
- prefix : '#include<intel-ipsec-mb.h>').split('"')[1]
-
- if (imb_ver == '') or (imb_ver.version_compare('<' + IMB_required_ver))
- reason = 'IPSec_MB version >= @0@ is required, found version @1@'.format(
- IMB_required_ver, imb_ver)
- build = false
- endif
-
-endif
-
-sources = files('rte_aesni_mb_pmd.c', 'rte_aesni_mb_pmd_ops.c')
-deps += ['bus_vdev', 'net', 'security']
deleted file mode 100644
@@ -1,3 +0,0 @@
-DPDK_22 {
- local: *;
-};
@@ -105,6 +105,7 @@ ipsec_mb_create(struct rte_vdev_device *vdev,
dev->dev_ops = ipsec_mb_pmds[pmd_type].ops;
dev->enqueue_burst = ipsec_mb_enqueue_burst;
dev->dequeue_burst = ipsec_mb_pmds[pmd_type].dequeue_burst;
+ dev->feature_flags = pmd_data->feature_flags;
if (pmd_data->dev_config) {
retval = (*pmd_data->dev_config)(dev);
@@ -116,8 +117,6 @@ ipsec_mb_create(struct rte_vdev_device *vdev,
}
}
- dev->feature_flags = pmd_data->feature_flags;
-
switch (vector_mode) {
case IPSEC_MB_AVX512:
dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
@@ -165,6 +164,10 @@ ipsec_mb_remove(struct rte_vdev_device *vdev)
rte_free(cryptodev->security_ctx);
cryptodev->security_ctx = NULL;
}
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+ rte_free(cryptodev->security_ctx);
+ cryptodev->security_ctx = NULL;
+#endif
for (qp_id = 0; qp_id < cryptodev->data->nb_queue_pairs; qp_id++)
ipsec_mb_qp_release(cryptodev, qp_id);
@@ -34,6 +34,9 @@ extern enum ipsec_mb_vector_mode vector_mode;
/** IMB_MGR instances, one per thread */
extern RTE_DEFINE_PER_LCORE(IMB_MGR *, mb_mgr);
+#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb
+/**< IPSEC Multi buffer aesni_mb PMD device name */
+
/** PMD LOGTYPE DRIVER, common to all PMDs */
extern int ipsec_mb_logtype_driver;
#define IPSEC_MB_LOG(level, fmt, ...) \
@@ -42,6 +45,7 @@ extern int ipsec_mb_logtype_driver;
/** All supported device types */
enum ipsec_mb_pmd_types {
+ IPSEC_MB_PMD_TYPE_AESNI_MB = 0,
IPSEC_MB_N_PMD_TYPES
};
@@ -60,10 +64,18 @@ enum ipsec_mb_operation {
IPSEC_MB_OP_NOT_SUPPORTED
};
+extern uint8_t pmd_driver_id_aesni_mb;
+
/** Helper function. Gets driver ID based on PMD type */
static __rte_always_inline uint8_t
-ipsec_mb_get_driver_id(__rte_unused enum ipsec_mb_pmd_types pmd_type)
+ipsec_mb_get_driver_id(enum ipsec_mb_pmd_types pmd_type)
{
+ switch (pmd_type) {
+ case IPSEC_MB_PMD_TYPE_AESNI_MB:
+ return pmd_driver_id_aesni_mb;
+ default:
+ break;
+ }
return UINT8_MAX;
}
@@ -136,6 +148,135 @@ get_per_thread_mb_mgr(void)
return RTE_PER_LCORE(mb_mgr);
}
+/** Helper function. Gets mode and chained xforms from the xform */
+static __rte_always_inline int
+ipsec_mb_parse_xform(const struct rte_crypto_sym_xform *xform,
+ enum ipsec_mb_operation *mode,
+ const struct rte_crypto_sym_xform **auth_xform,
+ const struct rte_crypto_sym_xform **cipher_xform,
+ const struct rte_crypto_sym_xform **aead_xform)
+{
+ const struct rte_crypto_sym_xform *next = xform->next;
+
+ if (xform == NULL) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+ if (next == NULL) {
+ if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+ *mode = IPSEC_MB_OP_ENCRYPT_ONLY;
+ *cipher_xform = xform;
+ *auth_xform = NULL;
+ return 0;
+ }
+ *mode = IPSEC_MB_OP_DECRYPT_ONLY;
+ *cipher_xform = xform;
+ *auth_xform = NULL;
+ return 0;
+ }
+
+ if (next->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+ if (next->auth.op != RTE_CRYPTO_AUTH_OP_GENERATE) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ *mode = IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN;
+ *cipher_xform = xform;
+ *auth_xform = xform->next;
+ return 0;
+ }
+ if (next->auth.op != RTE_CRYPTO_AUTH_OP_VERIFY) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ *mode = IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY;
+ *cipher_xform = xform;
+ *auth_xform = xform->next;
+ return 0;
+ }
+
+ if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+ if (next == NULL) {
+ if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
+ *mode = IPSEC_MB_OP_HASH_GEN_ONLY;
+ *auth_xform = xform;
+ *cipher_xform = NULL;
+ return 0;
+ }
+ *mode = IPSEC_MB_OP_HASH_VERIFY_ONLY;
+ *auth_xform = xform;
+ *cipher_xform = NULL;
+ return 0;
+ }
+
+ if (next->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
+ if (next->cipher.op != RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ *mode = IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT;
+ *auth_xform = xform;
+ *cipher_xform = xform->next;
+ return 0;
+ }
+ if (next->cipher.op != RTE_CRYPTO_CIPHER_OP_DECRYPT) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ *mode = IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT;
+ *auth_xform = xform;
+ *cipher_xform = xform->next;
+ return 0;
+ }
+
+ if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+ if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
+ /*
+ * CCM requires to hash first and cipher later
+ * when encrypting
+ */
+ if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
+ *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT;
+ *aead_xform = xform;
+ return 0;
+ } else {
+ *mode =
+ IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT;
+ *aead_xform = xform;
+ return 0;
+ }
+ } else {
+ if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
+ *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT;
+ *aead_xform = xform;
+ return 0;
+ }
+ *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT;
+ *aead_xform = xform;
+ return 0;
+ }
+ }
+
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+}
+
/** Device creation function */
int
ipsec_mb_create(struct rte_vdev_device *vdev,
@@ -23,5 +23,6 @@ endif
sources = files('ipsec_mb_private.c',
'ipsec_mb_ops.c',
+ 'pmd_aesni_mb.c'
)
deps += ['bus_vdev', 'net', 'security']
similarity index 60%
rename from drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
rename to drivers/crypto/ipsec_mb/pmd_aesni_mb.c
@@ -1,39 +1,14 @@
/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2017 Intel Corporation
+ * Copyright(c) 2015-2021 Intel Corporation
*/
-#include <intel-ipsec-mb.h>
-
-#include <rte_common.h>
-#include <rte_hexdump.h>
-#include <rte_cryptodev.h>
-#include <cryptodev_pmd.h>
-#include <rte_bus_vdev.h>
-#include <rte_malloc.h>
-#include <rte_cpuflags.h>
-#include <rte_per_lcore.h>
-#include <rte_ether.h>
-
-#include "aesni_mb_pmd_private.h"
-
-#define AES_CCM_DIGEST_MIN_LEN 4
-#define AES_CCM_DIGEST_MAX_LEN 16
-#define HMAC_MAX_BLOCK_SIZE 128
-static uint8_t cryptodev_driver_id;
-
-/*
- * Needed to support CPU-CRYPTO API (rte_cryptodev_sym_cpu_crypto_process),
- * as we still use JOB based API even for synchronous processing.
- */
-static RTE_DEFINE_PER_LCORE(MB_MGR *, sync_mb_mgr);
-
-typedef void (*hash_one_block_t)(const void *data, void *digest);
-typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
+#include "pmd_aesni_mb_priv.h"
/**
* Calculate the authentication pre-computes
*
- * @param one_block_hash Function pointer to calculate digest on ipad/opad
+ * @param one_block_hash Function pointer
+ * to calculate digest on ipad/opad
* @param ipad Inner pad output byte array
* @param opad Outer pad output byte array
* @param hkey Authentication key
@@ -46,7 +21,7 @@ calculate_auth_precomputes(hash_one_block_t one_block_hash,
const uint8_t *hkey, uint16_t hkey_len,
uint16_t blocksize)
{
- unsigned i, length;
+ uint32_t i, length;
uint8_t ipad_buf[blocksize] __rte_aligned(16);
uint8_t opad_buf[blocksize] __rte_aligned(16);
@@ -72,74 +47,18 @@ calculate_auth_precomputes(hash_one_block_t one_block_hash,
memset(opad_buf, 0, blocksize);
}
-/** Get xform chain order */
-static enum aesni_mb_operation
-aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
-{
- if (xform == NULL)
- return AESNI_MB_OP_NOT_SUPPORTED;
-
- if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
- if (xform->next == NULL)
- return AESNI_MB_OP_CIPHER_ONLY;
- if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
- return AESNI_MB_OP_CIPHER_HASH;
- }
-
- if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
- if (xform->next == NULL)
- return AESNI_MB_OP_HASH_ONLY;
- if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
- return AESNI_MB_OP_HASH_CIPHER;
- }
-#if IMB_VERSION_NUM > IMB_VERSION(0, 52, 0)
- if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
- if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
- /*
- * CCM requires to hash first and cipher later
- * when encrypting
- */
- if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
- return AESNI_MB_OP_AEAD_HASH_CIPHER;
- else
- return AESNI_MB_OP_AEAD_CIPHER_HASH;
- } else {
- if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
- return AESNI_MB_OP_AEAD_CIPHER_HASH;
- else
- return AESNI_MB_OP_AEAD_HASH_CIPHER;
- }
- }
-#else
- if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
- if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
- xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
- if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
- return AESNI_MB_OP_AEAD_CIPHER_HASH;
- else
- return AESNI_MB_OP_AEAD_HASH_CIPHER;
- }
- }
-#endif
-
- return AESNI_MB_OP_NOT_SUPPORTED;
-}
-
static inline int
-is_aead_algo(JOB_HASH_ALG hash_alg, JOB_CIPHER_MODE cipher_mode)
+is_aead_algo(IMB_HASH_ALG hash_alg, IMB_CIPHER_MODE cipher_mode)
{
-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
- return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 || hash_alg == AES_CCM ||
- (hash_alg == AES_GMAC && cipher_mode == GCM));
-#else
- return ((hash_alg == AES_GMAC && cipher_mode == GCM) ||
- hash_alg == AES_CCM);
-#endif
+ return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 ||
+ hash_alg == IMB_AUTH_AES_CCM ||
+ (hash_alg == IMB_AUTH_AES_GMAC &&
+ cipher_mode == IMB_CIPHER_GCM));
}
/** Set session authentication parameters */
static int
-aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
+aesni_mb_set_session_auth_parameters(const IMB_MGR *mb_mgr,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform)
{
@@ -149,12 +68,12 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
uint32_t auth_precompute = 1;
if (xform == NULL) {
- sess->auth.algo = NULL_HASH;
+ sess->auth.algo = IMB_AUTH_NULL;
return 0;
}
if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
- AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
+ IPSEC_MB_LOG(ERR, "Crypto xform struct not of type auth");
return -1;
}
@@ -170,12 +89,12 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
/* Set Authentication Parameters */
if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
- sess->auth.algo = AES_XCBC;
+ sess->auth.algo = IMB_AUTH_AES_XCBC;
uint16_t xcbc_mac_digest_len =
- get_truncated_digest_byte_length(AES_XCBC);
+ get_truncated_digest_byte_length(IMB_AUTH_AES_XCBC);
if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -189,12 +108,13 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
uint32_t dust[4*15];
- sess->auth.algo = AES_CMAC;
+ sess->auth.algo = IMB_AUTH_AES_CMAC;
- uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
+ uint16_t cmac_digest_len =
+ get_digest_byte_length(IMB_AUTH_AES_CMAC);
if (sess->auth.req_digest_len > cmac_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
/*
@@ -219,14 +139,15 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
- sess->cipher.direction = ENCRYPT;
- sess->chain_order = CIPHER_HASH;
+ sess->cipher.direction = IMB_DIR_ENCRYPT;
+ sess->chain_order = IMB_ORDER_CIPHER_HASH;
} else
- sess->cipher.direction = DECRYPT;
+ sess->cipher.direction = IMB_DIR_DECRYPT;
- sess->auth.algo = AES_GMAC;
- if (sess->auth.req_digest_len > get_digest_byte_length(AES_GMAC)) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ sess->auth.algo = IMB_AUTH_AES_GMAC;
+ if (sess->auth.req_digest_len >
+ get_digest_byte_length(IMB_AUTH_AES_GMAC)) {
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -234,36 +155,36 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
sess->iv.offset = xform->auth.iv.offset;
switch (xform->auth.key.length) {
- case AES_128_BYTES:
+ case IMB_KEY_128_BYTES:
IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
&sess->cipher.gcm_key);
- sess->cipher.key_length_in_bytes = AES_128_BYTES;
+ sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
break;
- case AES_192_BYTES:
+ case IMB_KEY_192_BYTES:
IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
&sess->cipher.gcm_key);
- sess->cipher.key_length_in_bytes = AES_192_BYTES;
+ sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
break;
- case AES_256_BYTES:
+ case IMB_KEY_256_BYTES:
IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
&sess->cipher.gcm_key);
- sess->cipher.key_length_in_bytes = AES_256_BYTES;
+ sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
break;
default:
- RTE_LOG(ERR, PMD, "failed to parse test type\n");
+ IPSEC_MB_LOG(ERR, "Invalid authentication key length\n");
return -EINVAL;
}
return 0;
}
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;
uint16_t zuc_eia3_digest_len =
- get_truncated_digest_byte_length(IMB_AUTH_ZUC_EIA3_BITLEN);
+ get_truncated_digest_byte_length(
+ IMB_AUTH_ZUC_EIA3_BITLEN);
if (sess->auth.req_digest_len != zuc_eia3_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -273,9 +194,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
} else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
sess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN;
uint16_t snow3g_uia2_digest_len =
- get_truncated_digest_byte_length(IMB_AUTH_SNOW3G_UIA2_BITLEN);
+ get_truncated_digest_byte_length(
+ IMB_AUTH_SNOW3G_UIA2_BITLEN);
if (sess->auth.req_digest_len != snow3g_uia2_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -288,7 +210,7 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
uint16_t kasumi_f9_digest_len =
get_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1);
if (sess->auth.req_digest_len != kasumi_f9_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -297,17 +219,17 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
&sess->auth.pKeySched_kasumi_auth);
return 0;
}
-#endif
switch (xform->auth.algo) {
case RTE_CRYPTO_AUTH_MD5_HMAC:
- sess->auth.algo = MD5;
+ sess->auth.algo = IMB_AUTH_MD5;
hash_oneblock_fn = mb_mgr->md5_one_block;
break;
case RTE_CRYPTO_AUTH_SHA1_HMAC:
- sess->auth.algo = SHA1;
+ sess->auth.algo = IMB_AUTH_HMAC_SHA_1;
hash_oneblock_fn = mb_mgr->sha1_one_block;
- if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
+ if (xform->auth.key.length > get_auth_algo_blocksize(
+ IMB_AUTH_HMAC_SHA_1)) {
IMB_SHA1(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
@@ -316,13 +238,14 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
}
break;
case RTE_CRYPTO_AUTH_SHA1:
- sess->auth.algo = PLAIN_SHA1;
+ sess->auth.algo = IMB_AUTH_SHA_1;
auth_precompute = 0;
break;
case RTE_CRYPTO_AUTH_SHA224_HMAC:
- sess->auth.algo = SHA_224;
+ sess->auth.algo = IMB_AUTH_HMAC_SHA_224;
hash_oneblock_fn = mb_mgr->sha224_one_block;
- if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
+ if (xform->auth.key.length > get_auth_algo_blocksize(
+ IMB_AUTH_HMAC_SHA_224)) {
IMB_SHA224(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
@@ -331,13 +254,14 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
}
break;
case RTE_CRYPTO_AUTH_SHA224:
- sess->auth.algo = PLAIN_SHA_224;
+ sess->auth.algo = IMB_AUTH_SHA_224;
auth_precompute = 0;
break;
case RTE_CRYPTO_AUTH_SHA256_HMAC:
- sess->auth.algo = SHA_256;
+ sess->auth.algo = IMB_AUTH_HMAC_SHA_256;
hash_oneblock_fn = mb_mgr->sha256_one_block;
- if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
+ if (xform->auth.key.length > get_auth_algo_blocksize(
+ IMB_AUTH_HMAC_SHA_256)) {
IMB_SHA256(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
@@ -346,13 +270,14 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
}
break;
case RTE_CRYPTO_AUTH_SHA256:
- sess->auth.algo = PLAIN_SHA_256;
+ sess->auth.algo = IMB_AUTH_SHA_256;
auth_precompute = 0;
break;
case RTE_CRYPTO_AUTH_SHA384_HMAC:
- sess->auth.algo = SHA_384;
+ sess->auth.algo = IMB_AUTH_HMAC_SHA_384;
hash_oneblock_fn = mb_mgr->sha384_one_block;
- if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
+ if (xform->auth.key.length > get_auth_algo_blocksize(
+ IMB_AUTH_HMAC_SHA_384)) {
IMB_SHA384(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
@@ -361,13 +286,14 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
}
break;
case RTE_CRYPTO_AUTH_SHA384:
- sess->auth.algo = PLAIN_SHA_384;
+ sess->auth.algo = IMB_AUTH_SHA_384;
auth_precompute = 0;
break;
case RTE_CRYPTO_AUTH_SHA512_HMAC:
- sess->auth.algo = SHA_512;
+ sess->auth.algo = IMB_AUTH_HMAC_SHA_512;
hash_oneblock_fn = mb_mgr->sha512_one_block;
- if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
+ if (xform->auth.key.length > get_auth_algo_blocksize(
+ IMB_AUTH_HMAC_SHA_512)) {
IMB_SHA512(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
@@ -376,11 +302,12 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
}
break;
case RTE_CRYPTO_AUTH_SHA512:
- sess->auth.algo = PLAIN_SHA_512;
+ sess->auth.algo = IMB_AUTH_SHA_512;
auth_precompute = 0;
break;
default:
- AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
+ IPSEC_MB_LOG(ERR,
+ "Unsupported authentication algorithm selection");
return -ENOTSUP;
}
uint16_t trunc_digest_size =
@@ -390,7 +317,7 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
if (sess->auth.req_digest_len > full_digest_size ||
sess->auth.req_digest_len == 0) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
@@ -424,73 +351,68 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
/** Set session cipher parameters */
static int
-aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
+aesni_mb_set_session_cipher_parameters(const IMB_MGR *mb_mgr,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform)
{
uint8_t is_aes = 0;
uint8_t is_3DES = 0;
uint8_t is_docsis = 0;
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
uint8_t is_zuc = 0;
uint8_t is_snow3g = 0;
uint8_t is_kasumi = 0;
-#endif
if (xform == NULL) {
- sess->cipher.mode = NULL_CIPHER;
+ sess->cipher.mode = IMB_CIPHER_NULL;
return 0;
}
if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
- AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
+ IPSEC_MB_LOG(ERR, "Crypto xform struct not of type cipher");
return -EINVAL;
}
/* Select cipher direction */
switch (xform->cipher.op) {
case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
- sess->cipher.direction = ENCRYPT;
+ sess->cipher.direction = IMB_DIR_ENCRYPT;
break;
case RTE_CRYPTO_CIPHER_OP_DECRYPT:
- sess->cipher.direction = DECRYPT;
+ sess->cipher.direction = IMB_DIR_DECRYPT;
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
+ IPSEC_MB_LOG(ERR, "Invalid cipher operation parameter");
return -EINVAL;
}
/* Select cipher mode */
switch (xform->cipher.algo) {
case RTE_CRYPTO_CIPHER_AES_CBC:
- sess->cipher.mode = CBC;
+ sess->cipher.mode = IMB_CIPHER_CBC;
is_aes = 1;
break;
case RTE_CRYPTO_CIPHER_AES_CTR:
- sess->cipher.mode = CNTR;
+ sess->cipher.mode = IMB_CIPHER_CNTR;
is_aes = 1;
break;
case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
- sess->cipher.mode = DOCSIS_SEC_BPI;
+ sess->cipher.mode = IMB_CIPHER_DOCSIS_SEC_BPI;
is_docsis = 1;
break;
case RTE_CRYPTO_CIPHER_DES_CBC:
- sess->cipher.mode = DES;
+ sess->cipher.mode = IMB_CIPHER_DES;
break;
case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
- sess->cipher.mode = DOCSIS_DES;
+ sess->cipher.mode = IMB_CIPHER_DOCSIS_DES;
break;
case RTE_CRYPTO_CIPHER_3DES_CBC:
- sess->cipher.mode = DES3;
+ sess->cipher.mode = IMB_CIPHER_DES3;
is_3DES = 1;
break;
-#if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
case RTE_CRYPTO_CIPHER_AES_ECB:
- sess->cipher.mode = ECB;
+ sess->cipher.mode = IMB_CIPHER_ECB;
is_aes = 1;
break;
-#endif
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
case RTE_CRYPTO_CIPHER_ZUC_EEA3:
sess->cipher.mode = IMB_CIPHER_ZUC_EEA3;
is_zuc = 1;
@@ -503,9 +425,8 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
sess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN;
is_kasumi = 1;
break;
-#endif
default:
- AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
+ IPSEC_MB_LOG(ERR, "Unsupported cipher mode parameter");
return -ENOTSUP;
}
@@ -516,46 +437,44 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
/* Check key length and choose key expansion function for AES */
if (is_aes) {
switch (xform->cipher.key.length) {
- case AES_128_BYTES:
- sess->cipher.key_length_in_bytes = AES_128_BYTES;
+ case IMB_KEY_128_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
- case AES_192_BYTES:
- sess->cipher.key_length_in_bytes = AES_192_BYTES;
+ case IMB_KEY_192_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
- case AES_256_BYTES:
- sess->cipher.key_length_in_bytes = AES_256_BYTES;
+ case IMB_KEY_256_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
} else if (is_docsis) {
switch (xform->cipher.key.length) {
- case AES_128_BYTES:
- sess->cipher.key_length_in_bytes = AES_128_BYTES;
+ case IMB_KEY_128_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- case AES_256_BYTES:
- sess->cipher.key_length_in_bytes = AES_256_BYTES;
+ case IMB_KEY_256_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
-#endif
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
} else if (is_3DES) {
@@ -597,15 +516,14 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 24;
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
} else if (is_zuc) {
if (xform->cipher.key.length != 16) {
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 16;
@@ -613,7 +531,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
16);
} else if (is_snow3g) {
if (xform->cipher.key.length != 16) {
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 16;
@@ -621,16 +539,15 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
&sess->cipher.pKeySched_snow3g_cipher);
} else if (is_kasumi) {
if (xform->cipher.key.length != 16) {
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 16;
IMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data,
&sess->cipher.pKeySched_kasumi_cipher);
-#endif
} else {
if (xform->cipher.key.length != 8) {
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 8;
@@ -647,21 +564,21 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
}
static int
-aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
+aesni_mb_set_session_aead_parameters(const IMB_MGR *mb_mgr,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform)
{
switch (xform->aead.op) {
case RTE_CRYPTO_AEAD_OP_ENCRYPT:
- sess->cipher.direction = ENCRYPT;
+ sess->cipher.direction = IMB_DIR_ENCRYPT;
sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
break;
case RTE_CRYPTO_AEAD_OP_DECRYPT:
- sess->cipher.direction = DECRYPT;
+ sess->cipher.direction = IMB_DIR_DECRYPT;
sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
break;
default:
- AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
+ IPSEC_MB_LOG(ERR, "Invalid aead operation parameter");
return -EINVAL;
}
@@ -675,151 +592,151 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
switch (xform->aead.algo) {
case RTE_CRYPTO_AEAD_AES_CCM:
- sess->cipher.mode = CCM;
- sess->auth.algo = AES_CCM;
+ sess->cipher.mode = IMB_CIPHER_CCM;
+ sess->auth.algo = IMB_AUTH_AES_CCM;
/* Check key length and choose key expansion function for AES */
switch (xform->aead.key.length) {
- case AES_128_BYTES:
- sess->cipher.key_length_in_bytes = AES_128_BYTES;
+ case IMB_KEY_128_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
- case AES_256_BYTES:
- sess->cipher.key_length_in_bytes = AES_256_BYTES;
+ case IMB_KEY_256_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
IMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data,
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
/* CCM digests must be between 4 and 16 and an even number */
if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
- sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
- (sess->auth.req_digest_len & 1) == 1) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
+ (sess->auth.req_digest_len & 1) == 1) {
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
break;
case RTE_CRYPTO_AEAD_AES_GCM:
- sess->cipher.mode = GCM;
- sess->auth.algo = AES_GMAC;
+ sess->cipher.mode = IMB_CIPHER_GCM;
+ sess->auth.algo = IMB_AUTH_AES_GMAC;
switch (xform->aead.key.length) {
- case AES_128_BYTES:
- sess->cipher.key_length_in_bytes = AES_128_BYTES;
+ case IMB_KEY_128_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
&sess->cipher.gcm_key);
break;
- case AES_192_BYTES:
- sess->cipher.key_length_in_bytes = AES_192_BYTES;
+ case IMB_KEY_192_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
&sess->cipher.gcm_key);
break;
- case AES_256_BYTES:
- sess->cipher.key_length_in_bytes = AES_256_BYTES;
+ case IMB_KEY_256_BYTES:
+ sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
&sess->cipher.gcm_key);
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
/* GCM digest size must be between 1 and 16 */
if (sess->auth.req_digest_len == 0 ||
sess->auth.req_digest_len > 16) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
break;
-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
sess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305;
sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;
if (xform->aead.key.length != 32) {
- AESNI_MB_LOG(ERR, "Invalid key length");
+ IPSEC_MB_LOG(ERR, "Invalid key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 32;
memcpy(sess->cipher.expanded_aes_keys.encode,
xform->aead.key.data, 32);
if (sess->auth.req_digest_len != 16) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
break;
-#endif
default:
- AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
+ IPSEC_MB_LOG(ERR, "Unsupported aead mode parameter");
return -ENOTSUP;
}
return 0;
}
-/** Parse crypto xform chain and set private session parameters */
-int
-aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
- struct aesni_mb_session *sess,
+/** Configure a aesni multi-buffer session from a crypto xform chain */
+static int
+aesni_mb_session_configure(IMB_MGR *mb_mgr,
+ void *priv_sess,
const struct rte_crypto_sym_xform *xform)
{
const struct rte_crypto_sym_xform *auth_xform = NULL;
const struct rte_crypto_sym_xform *cipher_xform = NULL;
const struct rte_crypto_sym_xform *aead_xform = NULL;
+ enum ipsec_mb_operation mode;
+ struct aesni_mb_session *sess = (struct aesni_mb_session *) priv_sess;
int ret;
+ ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
+ &cipher_xform, &aead_xform);
+ if (ret)
+ return ret;
+
/* Select Crypto operation - hash then cipher / cipher then hash */
- switch (aesni_mb_get_chain_order(xform)) {
- case AESNI_MB_OP_HASH_CIPHER:
- sess->chain_order = HASH_CIPHER;
- auth_xform = xform;
- cipher_xform = xform->next;
+ switch (mode) {
+ case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
+ sess->chain_order = IMB_ORDER_HASH_CIPHER;
break;
- case AESNI_MB_OP_CIPHER_HASH:
- sess->chain_order = CIPHER_HASH;
- auth_xform = xform->next;
- cipher_xform = xform;
+ case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
+ case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
+ sess->chain_order = IMB_ORDER_CIPHER_HASH;
break;
- case AESNI_MB_OP_HASH_ONLY:
- sess->chain_order = HASH_CIPHER;
- auth_xform = xform;
- cipher_xform = NULL;
+ case IPSEC_MB_OP_HASH_GEN_ONLY:
+ case IPSEC_MB_OP_HASH_VERIFY_ONLY:
+ case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
+ sess->chain_order = IMB_ORDER_HASH_CIPHER;
break;
- case AESNI_MB_OP_CIPHER_ONLY:
- /*
- * Multi buffer library operates only at two modes,
- * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
- * chain order depends on cipher operation: encryption is always
- * the first operation and decryption the last one.
- */
- if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
- sess->chain_order = CIPHER_HASH;
- else
- sess->chain_order = HASH_CIPHER;
- auth_xform = NULL;
- cipher_xform = xform;
+ /*
+ * Multi buffer library operates only at two modes,
+ * IMB_ORDER_CIPHER_HASH and IMB_ORDER_HASH_CIPHER.
+ * When doing ciphering only, chain order depends
+ * on cipher operation: encryption is always
+ * the first operation and decryption the last one.
+ */
+ case IPSEC_MB_OP_ENCRYPT_ONLY:
+ sess->chain_order = IMB_ORDER_CIPHER_HASH;
break;
- case AESNI_MB_OP_AEAD_CIPHER_HASH:
- sess->chain_order = CIPHER_HASH;
+ case IPSEC_MB_OP_DECRYPT_ONLY:
+ sess->chain_order = IMB_ORDER_HASH_CIPHER;
+ break;
+ case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
+ sess->chain_order = IMB_ORDER_CIPHER_HASH;
sess->aead.aad_len = xform->aead.aad_length;
- aead_xform = xform;
break;
- case AESNI_MB_OP_AEAD_HASH_CIPHER:
- sess->chain_order = HASH_CIPHER;
+ case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
+ sess->chain_order = IMB_ORDER_HASH_CIPHER;
sess->aead.aad_len = xform->aead.aad_length;
- aead_xform = xform;
break;
- case AESNI_MB_OP_NOT_SUPPORTED:
+ case IPSEC_MB_OP_NOT_SUPPORTED:
default:
- AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
+ IPSEC_MB_LOG(ERR,
+ "Unsupported operation chain order parameter");
return -ENOTSUP;
}
@@ -829,14 +746,15 @@ aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
+ IPSEC_MB_LOG(ERR,
+ "Invalid/unsupported authentication parameters");
return ret;
}
ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
cipher_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
+ IPSEC_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
return ret;
}
@@ -844,7 +762,8 @@ aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
aead_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
+ IPSEC_MB_LOG(ERR,
+ "Invalid/unsupported aead parameters");
return ret;
}
}
@@ -868,9 +787,9 @@ check_docsis_sec_session(struct rte_security_session_conf *conf)
crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
crypto_sym->cipher.algo ==
RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
- (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||
- crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&
- crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
+ (crypto_sym->cipher.key.length == IMB_KEY_128_BYTES ||
+ crypto_sym->cipher.key.length == IMB_KEY_256_BYTES) &&
+ crypto_sym->cipher.iv.length == IMB_AES_BLOCK_SIZE &&
crypto_sym->next == NULL) {
return 0;
}
@@ -882,9 +801,9 @@ check_docsis_sec_session(struct rte_security_session_conf *conf)
crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
crypto_sym->cipher.algo ==
RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
- (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||
- crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&
- crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
+ (crypto_sym->cipher.key.length == IMB_KEY_128_BYTES ||
+ crypto_sym->cipher.key.length == IMB_KEY_256_BYTES) &&
+ crypto_sym->cipher.iv.length == IMB_AES_BLOCK_SIZE &&
crypto_sym->next == NULL) {
return 0;
}
@@ -899,7 +818,7 @@ aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
struct rte_security_docsis_xform *xform)
{
if (xform == NULL) {
- AESNI_MB_LOG(ERR, "Invalid DOCSIS xform");
+ IPSEC_MB_LOG(ERR, "Invalid DOCSIS xform");
return -EINVAL;
}
@@ -911,7 +830,7 @@ aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
} else {
- AESNI_MB_LOG(ERR, "Unsupported DOCSIS direction");
+ IPSEC_MB_LOG(ERR, "Unsupported DOCSIS direction");
return -ENOTSUP;
}
@@ -925,135 +844,67 @@ aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
* Parse DOCSIS security session configuration and set private session
* parameters
*/
-int
+static int
aesni_mb_set_docsis_sec_session_parameters(
__rte_unused struct rte_cryptodev *dev,
struct rte_security_session_conf *conf,
void *sess)
{
+ IMB_MGR *mb_mgr = alloc_init_mb_mgr();
struct rte_security_docsis_xform *docsis_xform;
struct rte_crypto_sym_xform *cipher_xform;
- struct aesni_mb_session *aesni_sess = sess;
- struct aesni_mb_private *internals = dev->data->dev_private;
- int ret;
+ struct aesni_mb_session *ipsec_sess = sess;
+ int ret = 0;
+
+ if (!mb_mgr)
+ return -ENOMEM;
ret = check_docsis_sec_session(conf);
if (ret) {
- AESNI_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
- return ret;
+ IPSEC_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
+ goto error_exit;
}
switch (conf->docsis.direction) {
case RTE_SECURITY_DOCSIS_UPLINK:
- aesni_sess->chain_order = IMB_ORDER_CIPHER_HASH;
+ ipsec_sess->chain_order = IMB_ORDER_CIPHER_HASH;
docsis_xform = &conf->docsis;
cipher_xform = conf->crypto_xform;
break;
case RTE_SECURITY_DOCSIS_DOWNLINK:
- aesni_sess->chain_order = IMB_ORDER_HASH_CIPHER;
+ ipsec_sess->chain_order = IMB_ORDER_HASH_CIPHER;
cipher_xform = conf->crypto_xform;
docsis_xform = &conf->docsis;
break;
default:
- return -EINVAL;
+ IPSEC_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
+ ret = -EINVAL;
+ goto error_exit;
}
/* Default IV length = 0 */
- aesni_sess->iv.length = 0;
+ ipsec_sess->iv.length = 0;
- ret = aesni_mb_set_docsis_sec_session_auth_parameters(aesni_sess,
+ ret = aesni_mb_set_docsis_sec_session_auth_parameters(ipsec_sess,
docsis_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
- return -EINVAL;
+ IPSEC_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
+ goto error_exit;
}
- ret = aesni_mb_set_session_cipher_parameters(internals->mb_mgr,
- aesni_sess, cipher_xform);
+ ret = aesni_mb_set_session_cipher_parameters(mb_mgr,
+ ipsec_sess, cipher_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
- return -EINVAL;
+ IPSEC_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
+ goto error_exit;
}
- return 0;
-}
-#endif
-
-/**
- * burst enqueue, place crypto operations on ingress queue for processing.
- *
- * @param __qp Queue Pair to process
- * @param ops Crypto operations for processing
- * @param nb_ops Number of crypto operations for processing
- *
- * @return
- * - Number of crypto operations enqueued
- */
-static uint16_t
-aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
- uint16_t nb_ops)
-{
- struct aesni_mb_qp *qp = __qp;
-
- unsigned int nb_enqueued;
-
- nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
- (void **)ops, nb_ops, NULL);
-
- qp->stats.enqueued_count += nb_enqueued;
-
- return nb_enqueued;
+error_exit:
+ free_mb_mgr(mb_mgr);
+ return ret;
}
-
-/** Get multi buffer session */
-static inline struct aesni_mb_session *
-get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
-{
- struct aesni_mb_session *sess = NULL;
-
- if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
- if (likely(op->sym->session != NULL))
- sess = (struct aesni_mb_session *)
- get_sym_session_private_data(
- op->sym->session,
- cryptodev_driver_id);
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
- } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
- if (likely(op->sym->sec_session != NULL))
- sess = (struct aesni_mb_session *)
- get_sec_session_private_data(
- op->sym->sec_session);
#endif
- } else {
- void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
- void *_sess_private_data = NULL;
-
- if (_sess == NULL)
- return NULL;
-
- if (rte_mempool_get(qp->sess_mp_priv,
- (void **)&_sess_private_data))
- return NULL;
-
- sess = (struct aesni_mb_session *)_sess_private_data;
-
- if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
- sess, op->sym->xform) != 0)) {
- rte_mempool_put(qp->sess_mp, _sess);
- rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
- sess = NULL;
- }
- op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
- set_sym_session_private_data(op->sym->session,
- cryptodev_driver_id, _sess_private_data);
- }
-
- if (unlikely(sess == NULL))
- op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-
- return sess;
-}
static inline uint64_t
auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
@@ -1065,7 +916,7 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
uint32_t cipher_end, auth_end;
/* Only cipher then hash needs special calculation. */
- if (!oop || session->chain_order != CIPHER_HASH)
+ if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH)
return op->sym->auth.data.offset;
m_src = op->sym->m_src;
@@ -1106,7 +957,7 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
}
static inline void
-set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
+set_cpu_mb_job_params(IMB_JOB *job, struct aesni_mb_session *session,
union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)
@@ -1118,82 +969,71 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
job->cipher_direction = session->cipher.direction;
job->cipher_mode = session->cipher.mode;
- job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+ job->key_len_in_bytes = session->cipher.key_length_in_bytes;
/* Set authentication parameters */
job->hash_alg = session->auth.algo;
job->iv = iv->va;
switch (job->hash_alg) {
- case AES_XCBC:
+ case IMB_AUTH_AES_XCBC:
job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
job->u.XCBC._k2 = session->auth.xcbc.k2;
job->u.XCBC._k3 = session->auth.xcbc.k3;
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
break;
- case AES_CCM:
+ case IMB_AUTH_AES_CCM:
job->u.CCM.aad = (uint8_t *)aad->va + 18;
job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
job->iv++;
break;
- case AES_CMAC:
+ case IMB_AUTH_AES_CMAC:
job->u.CMAC._key_expanded = session->auth.cmac.expkey;
job->u.CMAC._skey1 = session->auth.cmac.skey1;
job->u.CMAC._skey2 = session->auth.cmac.skey2;
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
break;
- case AES_GMAC:
- if (session->cipher.mode == GCM) {
+ case IMB_AUTH_AES_GMAC:
+ if (session->cipher.mode == IMB_CIPHER_GCM) {
job->u.GCM.aad = aad->va;
job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
} else {
/* For GMAC */
job->u.GCM.aad = buf;
job->u.GCM.aad_len_in_bytes = len;
- job->cipher_mode = GCM;
+ job->cipher_mode = IMB_CIPHER_GCM;
}
- job->aes_enc_key_expanded = &session->cipher.gcm_key;
- job->aes_dec_key_expanded = &session->cipher.gcm_key;
+ job->enc_keys = &session->cipher.gcm_key;
+ job->dec_keys = &session->cipher.gcm_key;
break;
-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
case IMB_AUTH_CHACHA20_POLY1305:
job->u.CHACHA20_POLY1305.aad = aad->va;
- job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
- job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes =
+ session->aead.aad_len;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.encode;
break;
-#endif
default:
job->u.HMAC._hashed_auth_key_xor_ipad =
session->auth.pads.inner;
job->u.HMAC._hashed_auth_key_xor_opad =
session->auth.pads.outer;
- if (job->cipher_mode == DES3) {
- job->aes_enc_key_expanded =
- session->cipher.exp_3des_keys.ks_ptr;
- job->aes_dec_key_expanded =
- session->cipher.exp_3des_keys.ks_ptr;
+ if (job->cipher_mode == IMB_CIPHER_DES3) {
+ job->enc_keys = session->cipher.exp_3des_keys.ks_ptr;
+ job->dec_keys = session->cipher.exp_3des_keys.ks_ptr;
} else {
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
}
}
@@ -1214,7 +1054,8 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;
job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;
job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;
- if (job->hash_alg == AES_GMAC && session->cipher.mode != GCM) {
+ if (job->hash_alg == IMB_AUTH_AES_GMAC &&
+ session->cipher.mode != IMB_CIPHER_GCM) {
job->msg_len_to_hash_in_bytes = 0;
job->msg_len_to_cipher_in_bytes = 0;
} else {
@@ -1228,26 +1069,28 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
}
/**
- * Process a crypto operation and complete a JOB_AES_HMAC job structure for
+ * Process a crypto operation and complete a IMB_JOB job structure for
* submission to the multi buffer library for processing.
*
- * @param qp queue pair
- * @param job JOB_AES_HMAC structure to fill
- * @param m mbuf to process
+ * @param qp queue pair
+ * @param job IMB_JOB structure to fill
+ * @param op crypto op to process
+ * @param digest_idx ID for digest to use
*
* @return
- * - Completed JOB_AES_HMAC structure pointer on success
- * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
+ * - 0 on success, the IMB_JOB will be filled
+ * - -1 if invalid session, IMB_JOB will not be filled
*/
static inline int
-set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
+set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
struct rte_crypto_op *op, uint8_t *digest_idx)
{
struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
+ struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
struct aesni_mb_session *session;
uint32_t m_offset, oop;
- session = get_session(qp, op);
+ session = ipsec_mb_get_session_private(qp, op);
if (session == NULL) {
op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
return -1;
@@ -1260,7 +1103,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->cipher_direction = session->cipher.direction;
job->cipher_mode = session->cipher.mode;
- job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+ job->key_len_in_bytes = session->cipher.key_length_in_bytes;
/* Set authentication parameters */
job->hash_alg = session->auth.algo;
@@ -1268,38 +1111,32 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
const int aead = is_aead_algo(job->hash_alg, job->cipher_mode);
switch (job->hash_alg) {
- case AES_XCBC:
+ case IMB_AUTH_AES_XCBC:
job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
job->u.XCBC._k2 = session->auth.xcbc.k2;
job->u.XCBC._k3 = session->auth.xcbc.k3;
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
break;
- case AES_CCM:
+ case IMB_AUTH_AES_CCM:
job->u.CCM.aad = op->sym->aead.aad.data + 18;
job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
break;
- case AES_CMAC:
+ case IMB_AUTH_AES_CMAC:
job->u.CMAC._key_expanded = session->auth.cmac.expkey;
job->u.CMAC._skey1 = session->auth.cmac.skey1;
job->u.CMAC._skey2 = session->auth.cmac.skey2;
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
break;
- case AES_GMAC:
- if (session->cipher.mode == GCM) {
+ case IMB_AUTH_AES_GMAC:
+ if (session->cipher.mode == IMB_CIPHER_GCM) {
job->u.GCM.aad = op->sym->aead.aad.data;
job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
} else {
@@ -1307,48 +1144,46 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
uint8_t *, op->sym->auth.data.offset);
job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
- job->cipher_mode = GCM;
+ job->cipher_mode = IMB_CIPHER_GCM;
}
- job->aes_enc_key_expanded = &session->cipher.gcm_key;
- job->aes_dec_key_expanded = &session->cipher.gcm_key;
+ job->enc_keys = &session->cipher.gcm_key;
+ job->dec_keys = &session->cipher.gcm_key;
break;
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
case IMB_AUTH_ZUC_EIA3_BITLEN:
job->u.ZUC_EIA3._key = session->auth.zuc_auth_key;
job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
session->auth_iv.offset);
break;
case IMB_AUTH_SNOW3G_UIA2_BITLEN:
- job->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth;
- job->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ job->u.SNOW3G_UIA2._key = (void *)
+ &session->auth.pKeySched_snow3g_auth;
+ job->u.SNOW3G_UIA2._iv =
+ rte_crypto_op_ctod_offset(op, uint8_t *,
session->auth_iv.offset);
break;
case IMB_AUTH_KASUMI_UIA1:
- job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth;
+ job->u.KASUMI_UIA1._key = (void *)
+ &session->auth.pKeySched_kasumi_auth;
break;
-#endif
-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
case IMB_AUTH_CHACHA20_POLY1305:
job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
- job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
- job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes =
+ session->aead.aad_len;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.encode;
break;
-#endif
default:
- job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
- job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
-
- if (job->cipher_mode == DES3) {
- job->aes_enc_key_expanded =
- session->cipher.exp_3des_keys.ks_ptr;
- job->aes_dec_key_expanded =
- session->cipher.exp_3des_keys.ks_ptr;
+ job->u.HMAC._hashed_auth_key_xor_ipad =
+ session->auth.pads.inner;
+ job->u.HMAC._hashed_auth_key_xor_opad =
+ session->auth.pads.outer;
+
+ if (job->cipher_mode == IMB_CIPHER_DES3) {
+ job->enc_keys = session->cipher.exp_3des_keys.ks_ptr;
+ job->dec_keys = session->cipher.exp_3des_keys.ks_ptr;
} else {
- job->aes_enc_key_expanded =
- session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded =
- session->cipher.expanded_aes_keys.decode;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
}
}
@@ -1357,10 +1192,9 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
else
m_offset = op->sym->cipher.data.offset;
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
- job->aes_enc_key_expanded = session->cipher.zuc_cipher_key;
- job->aes_dec_key_expanded = session->cipher.zuc_cipher_key;
+ job->enc_keys = session->cipher.zuc_cipher_key;
+ job->dec_keys = session->cipher.zuc_cipher_key;
} else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
m_offset = 0;
@@ -1368,7 +1202,6 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->enc_keys = &session->cipher.pKeySched_kasumi_cipher;
m_offset = 0;
}
-#endif
if (!op->sym->m_dst) {
/* in-place operation */
@@ -1385,19 +1218,21 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
}
/* Set digest output location */
- if (job->hash_alg != NULL_HASH &&
+ if (job->hash_alg != IMB_AUTH_NULL &&
session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
- job->auth_tag_output = qp->temp_digests[*digest_idx];
- *digest_idx = (*digest_idx + 1) % MAX_JOBS;
+ job->auth_tag_output = qp_data->temp_digests[*digest_idx];
+ *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
} else {
if (aead)
job->auth_tag_output = op->sym->aead.digest.data;
else
job->auth_tag_output = op->sym->auth.digest.data;
- if (session->auth.req_digest_len != session->auth.gen_digest_len) {
- job->auth_tag_output = qp->temp_digests[*digest_idx];
- *digest_idx = (*digest_idx + 1) % MAX_JOBS;
+ if (session->auth.req_digest_len !=
+ session->auth.gen_digest_len) {
+ job->auth_tag_output =
+ qp_data->temp_digests[*digest_idx];
+ *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
}
}
/*
@@ -1416,7 +1251,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
switch (job->hash_alg) {
- case AES_CCM:
+ case IMB_AUTH_AES_CCM:
job->cipher_start_src_offset_in_bytes =
op->sym->aead.data.offset;
job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
@@ -1427,8 +1262,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
session->iv.offset + 1);
break;
- case AES_GMAC:
- if (session->cipher.mode == GCM) {
+ case IMB_AUTH_AES_GMAC:
+ if (session->cipher.mode == IMB_CIPHER_GCM) {
job->cipher_start_src_offset_in_bytes =
op->sym->aead.data.offset;
job->hash_start_src_offset_in_bytes =
@@ -1450,10 +1285,11 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
session->iv.offset);
break;
-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
case IMB_AUTH_CHACHA20_POLY1305:
- job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset;
- job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->cipher_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
+ job->hash_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
job->msg_len_to_cipher_in_bytes =
op->sym->aead.data.length;
job->msg_len_to_hash_in_bytes =
@@ -1462,7 +1298,6 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
session->iv.offset);
break;
-#endif
default:
/* For SNOW3G, length and offsets are already in bits */
job->cipher_start_src_offset_in_bytes =
@@ -1477,12 +1312,10 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
session->iv.offset);
}
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
job->msg_len_to_cipher_in_bytes >>= 3;
else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)
job->msg_len_to_hash_in_bytes >>= 3;
-#endif
/* Set user data to be crypto operation data struct */
job->user_data = op;
@@ -1493,23 +1326,29 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
/**
* Process a crypto operation containing a security op and complete a
- * JOB_AES_HMAC job structure for submission to the multi buffer library for
+ * IMB_JOB job structure for submission to the multi buffer library for
* processing.
*/
static inline int
-set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
- struct rte_crypto_op *op, uint8_t *digest_idx)
+set_sec_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
+ struct rte_crypto_op *op, uint8_t *digest_idx)
{
+ struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
struct rte_mbuf *m_src, *m_dst;
struct rte_crypto_sym_op *sym;
- struct aesni_mb_session *session;
+ struct aesni_mb_session *session = NULL;
- session = get_session(qp, op);
- if (unlikely(session == NULL)) {
+ if (unlikely(op->sess_type != RTE_CRYPTO_OP_SECURITY_SESSION)) {
op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
return -1;
}
+ session = (struct aesni_mb_session *)
+ get_sec_session_private_data(op->sym->sec_session);
+ if (unlikely(session == NULL)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+ return -1;
+ }
/* Only DOCSIS protocol operations supported now */
if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||
session->auth.algo != IMB_AUTH_DOCSIS_CRC32) {
@@ -1536,9 +1375,9 @@ set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->cipher_direction = session->cipher.direction;
job->cipher_mode = session->cipher.mode;
- job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
- job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
+ job->key_len_in_bytes = session->cipher.key_length_in_bytes;
+ job->enc_keys = session->cipher.expanded_aes_keys.encode;
+ job->dec_keys = session->cipher.expanded_aes_keys.decode;
/* Set IV parameters */
job->iv_len_in_bytes = session->iv.length;
@@ -1548,8 +1387,8 @@ set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->hash_alg = session->auth.algo;
/* Set digest output location */
- job->auth_tag_output = qp->temp_digests[*digest_idx];
- *digest_idx = (*digest_idx + 1) % MAX_JOBS;
+ job->auth_tag_output = qp_data->temp_digests[*digest_idx];
+ *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
/* Set digest length */
job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
@@ -1571,7 +1410,7 @@ set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
}
static inline void
-verify_docsis_sec_crc(JOB_AES_HMAC *job, uint8_t *status)
+verify_docsis_sec_crc(IMB_JOB *job, uint8_t *status)
{
uint16_t crc_offset;
uint8_t *crc;
@@ -1591,7 +1430,7 @@ verify_docsis_sec_crc(JOB_AES_HMAC *job, uint8_t *status)
#endif
static inline void
-verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
+verify_digest(IMB_JOB *job, void *digest, uint16_t len, uint8_t *status)
{
/* Verify digest if required */
if (memcmp(job->auth_tag_output, digest, len) != 0)
@@ -1599,7 +1438,7 @@ verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
}
static inline void
-generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
+generate_digest(IMB_JOB *job, struct rte_crypto_op *op,
struct aesni_mb_session *sess)
{
/* No extra copy needed */
@@ -1617,18 +1456,20 @@ generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
/**
* Process a completed job and return rte_mbuf which job processed
*
- * @param qp Queue Pair to process
- * @param job JOB_AES_HMAC job to process
+ * @param qp Queue Pair to process
+ * @param job IMB_JOB job to process
*
* @return
* - Returns processed crypto operation.
* - Returns NULL on invalid job
*/
static inline struct rte_crypto_op *
-post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
+post_process_mb_job(struct ipsec_mb_qp *qp, IMB_JOB *job)
{
struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
struct aesni_mb_session *sess = NULL;
+ uint32_t driver_id = ipsec_mb_get_driver_id(
+ IPSEC_MB_PMD_TYPE_AESNI_MB);
#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
uint8_t is_docsis_sec = 0;
@@ -1644,7 +1485,7 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
#endif
{
sess = get_sym_session_private_data(op->sym->session,
- cryptodev_driver_id);
+ driver_id);
}
if (unlikely(sess == NULL)) {
@@ -1654,14 +1495,15 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
switch (job->status) {
- case STS_COMPLETED:
+ case IMB_STATUS_COMPLETED:
op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
- if (job->hash_alg == NULL_HASH)
+ if (job->hash_alg == IMB_AUTH_NULL)
break;
if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
- if (is_aead_algo(job->hash_alg, sess->cipher.mode))
+ if (is_aead_algo(job->hash_alg,
+ sess->cipher.mode))
verify_digest(job,
op->sym->aead.digest.data,
sess->auth.req_digest_len,
@@ -1699,30 +1541,34 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
}
static inline void
-post_process_mb_sync_job(JOB_AES_HMAC *job)
+post_process_mb_sync_job(IMB_JOB *job)
{
uint32_t *st;
st = job->user_data;
- st[0] = (job->status == STS_COMPLETED) ? 0 : EBADMSG;
+ st[0] = (job->status == IMB_STATUS_COMPLETED) ? 0 : EBADMSG;
}
/**
- * Process a completed JOB_AES_HMAC job and keep processing jobs until
+ * Process a completed IMB_JOB job and keep processing jobs until
* get_completed_job return NULL
*
* @param qp Queue Pair to process
- * @param job JOB_AES_HMAC job
+ * @param mb_mgr IMB_MGR to use
+ * @param job IMB_JOB job
+ * @param ops crypto ops to fill
+ * @param nb_ops number of crypto ops
*
* @return
* - Number of processed jobs
*/
static unsigned
-handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
- struct rte_crypto_op **ops, uint16_t nb_ops)
+handle_completed_jobs(struct ipsec_mb_qp *qp, IMB_MGR *mb_mgr,
+ IMB_JOB *job, struct rte_crypto_op **ops,
+ uint16_t nb_ops)
{
struct rte_crypto_op *op = NULL;
- unsigned processed_jobs = 0;
+ uint16_t processed_jobs = 0;
while (job != NULL) {
op = post_process_mb_job(qp, job);
@@ -1737,14 +1583,14 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
if (processed_jobs == nb_ops)
break;
- job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
+ job = IMB_GET_COMPLETED_JOB(mb_mgr);
}
return processed_jobs;
}
static inline uint32_t
-handle_completed_sync_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr)
+handle_completed_sync_jobs(IMB_JOB *job, IMB_MGR *mb_mgr)
{
uint32_t i;
@@ -1755,37 +1601,37 @@ handle_completed_sync_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr)
}
static inline uint32_t
-flush_mb_sync_mgr(MB_MGR *mb_mgr)
+flush_mb_sync_mgr(IMB_MGR *mb_mgr)
{
- JOB_AES_HMAC *job;
+ IMB_JOB *job;
job = IMB_FLUSH_JOB(mb_mgr);
return handle_completed_sync_jobs(job, mb_mgr);
}
static inline uint16_t
-flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
- uint16_t nb_ops)
+flush_mb_mgr(struct ipsec_mb_qp *qp, IMB_MGR *mb_mgr,
+ struct rte_crypto_op **ops, uint16_t nb_ops)
{
int processed_ops = 0;
/* Flush the remaining jobs */
- JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
+ IMB_JOB *job = IMB_FLUSH_JOB(mb_mgr);
if (job)
- processed_ops += handle_completed_jobs(qp, job,
+ processed_ops += handle_completed_jobs(qp, mb_mgr, job,
&ops[processed_ops], nb_ops - processed_ops);
return processed_ops;
}
-static inline JOB_AES_HMAC *
-set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
+static inline IMB_JOB *
+set_job_null_op(IMB_JOB *job, struct rte_crypto_op *op)
{
- job->chain_order = HASH_CIPHER;
- job->cipher_mode = NULL_CIPHER;
- job->hash_alg = NULL_HASH;
- job->cipher_direction = DECRYPT;
+ job->chain_order = IMB_ORDER_HASH_CIPHER;
+ job->cipher_mode = IMB_CIPHER_NULL;
+ job->hash_alg = IMB_AUTH_NULL;
+ job->cipher_direction = IMB_DIR_DECRYPT;
/* Set user data to be crypto operation data struct */
job->user_data = op;
@@ -1794,39 +1640,39 @@ set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
}
static uint16_t
-aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
uint16_t nb_ops)
{
- struct aesni_mb_qp *qp = queue_pair;
-
+ struct ipsec_mb_qp *qp = queue_pair;
+ IMB_MGR *mb_mgr = qp->mb_mgr;
struct rte_crypto_op *op;
- JOB_AES_HMAC *job;
-
+ IMB_JOB *job;
int retval, processed_jobs = 0;
- if (unlikely(nb_ops == 0))
+ if (unlikely(nb_ops == 0 || mb_mgr == NULL))
return 0;
uint8_t digest_idx = qp->digest_idx;
+
do {
/* Get next free mb job struct from mb manager */
- job = IMB_GET_NEXT_JOB(qp->mb_mgr);
+ job = IMB_GET_NEXT_JOB(mb_mgr);
if (unlikely(job == NULL)) {
/* if no free mb job structs we need to flush mb_mgr */
- processed_jobs += flush_mb_mgr(qp,
+ processed_jobs += flush_mb_mgr(qp, mb_mgr,
&ops[processed_jobs],
nb_ops - processed_jobs);
if (nb_ops == processed_jobs)
break;
- job = IMB_GET_NEXT_JOB(qp->mb_mgr);
+ job = IMB_GET_NEXT_JOB(mb_mgr);
}
/*
* Get next operation to process from ingress queue.
- * There is no need to return the job to the MB_MGR
- * if there are no more operations to process, since the MB_MGR
+ * There is no need to return the job to the IMB_MGR
+ * if there are no more operations to process, since the IMB_MGR
* can use that pointer again in next get_next calls.
*/
retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
@@ -1839,7 +1685,8 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
&digest_idx);
else
#endif
- retval = set_mb_job_params(job, qp, op, &digest_idx);
+ retval = set_mb_job_params(job, qp, op,
+ &digest_idx);
if (unlikely(retval != 0)) {
qp->stats.dequeue_err_count++;
@@ -1848,17 +1695,17 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
/* Submit job to multi-buffer for processing */
#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
- job = IMB_SUBMIT_JOB(qp->mb_mgr);
+ job = IMB_SUBMIT_JOB(mb_mgr);
#else
- job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
+ job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
#endif
/*
* If submit returns a processed job then handle it,
* before submitting subsequent jobs
*/
if (job)
- processed_jobs += handle_completed_jobs(qp, job,
- &ops[processed_jobs],
+ processed_jobs += handle_completed_jobs(qp, mb_mgr,
+ job, &ops[processed_jobs],
nb_ops - processed_jobs);
} while (processed_jobs < nb_ops);
@@ -1866,44 +1713,16 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
qp->digest_idx = digest_idx;
if (processed_jobs < 1)
- processed_jobs += flush_mb_mgr(qp,
+ processed_jobs += flush_mb_mgr(qp, mb_mgr,
&ops[processed_jobs],
nb_ops - processed_jobs);
return processed_jobs;
}
-static MB_MGR *
-alloc_init_mb_mgr(enum aesni_mb_vector_mode vector_mode)
-{
- MB_MGR *mb_mgr = alloc_mb_mgr(0);
- if (mb_mgr == NULL)
- return NULL;
-
- switch (vector_mode) {
- case RTE_AESNI_MB_SSE:
- init_mb_mgr_sse(mb_mgr);
- break;
- case RTE_AESNI_MB_AVX:
- init_mb_mgr_avx(mb_mgr);
- break;
- case RTE_AESNI_MB_AVX2:
- init_mb_mgr_avx2(mb_mgr);
- break;
- case RTE_AESNI_MB_AVX512:
- init_mb_mgr_avx512(mb_mgr);
- break;
- default:
- AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
- free_mb_mgr(mb_mgr);
- return NULL;
- }
-
- return mb_mgr;
-}
static inline void
-aesni_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
+ipsec_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
{
uint32_t i;
@@ -1916,14 +1735,14 @@ check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)
{
/* no multi-seg support with current AESNI-MB PMD */
if (sgl->num != 1)
- return ENOTSUP;
+ return -ENOTSUP;
else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)
- return EINVAL;
+ return -EINVAL;
return 0;
}
-static inline JOB_AES_HMAC *
-submit_sync_job(MB_MGR *mb_mgr)
+static inline IMB_JOB *
+submit_sync_job(IMB_MGR *mb_mgr)
{
#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
return IMB_SUBMIT_JOB(mb_mgr);
@@ -1966,42 +1785,31 @@ verify_sync_dgst(struct rte_crypto_sym_vec *vec,
return k;
}
-uint32_t
-aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
+static uint32_t
+aesni_mb_process_bulk(struct rte_cryptodev *dev,
struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
struct rte_crypto_sym_vec *vec)
{
int32_t ret;
uint32_t i, j, k, len;
void *buf;
- JOB_AES_HMAC *job;
- MB_MGR *mb_mgr;
- struct aesni_mb_private *priv;
+ IMB_JOB *job;
+ IMB_MGR *mb_mgr;
struct aesni_mb_session *s;
uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];
s = get_sym_session_private_data(sess, dev->driver_id);
if (s == NULL) {
- aesni_mb_fill_error_code(vec, EINVAL);
+ ipsec_mb_fill_error_code(vec, EINVAL);
return 0;
}
/* get per-thread MB MGR, create one if needed */
- mb_mgr = RTE_PER_LCORE(sync_mb_mgr);
- if (mb_mgr == NULL) {
-
- priv = dev->data->dev_private;
- mb_mgr = alloc_init_mb_mgr(priv->vector_mode);
- if (mb_mgr == NULL) {
- aesni_mb_fill_error_code(vec, ENOMEM);
- return 0;
- }
- RTE_PER_LCORE(sync_mb_mgr) = mb_mgr;
- }
+ mb_mgr = get_per_thread_mb_mgr();
+ if (unlikely(mb_mgr == NULL))
+ return 0;
for (i = 0, j = 0, k = 0; i != vec->num; i++) {
-
-
ret = check_crypto_sgl(sofs, vec->sgl + i);
if (ret != 0) {
vec->status[i] = ret;
@@ -2047,186 +1855,172 @@ aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
return k;
}
-static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
+struct rte_cryptodev_ops aesni_mb_pmd_ops = {
+ .dev_configure = ipsec_mb_config,
+ .dev_start = ipsec_mb_start,
+ .dev_stop = ipsec_mb_stop,
+ .dev_close = ipsec_mb_close,
-static uint64_t
-vec_mode_to_flags(enum aesni_mb_vector_mode mode)
-{
- switch (mode) {
- case RTE_AESNI_MB_SSE:
- return RTE_CRYPTODEV_FF_CPU_SSE;
- case RTE_AESNI_MB_AVX:
- return RTE_CRYPTODEV_FF_CPU_AVX;
- case RTE_AESNI_MB_AVX2:
- return RTE_CRYPTODEV_FF_CPU_AVX2;
- case RTE_AESNI_MB_AVX512:
- return RTE_CRYPTODEV_FF_CPU_AVX512;
- default:
- AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", mode);
- return 0;
- }
-}
+ .stats_get = ipsec_mb_stats_get,
+ .stats_reset = ipsec_mb_stats_reset,
-static int
-cryptodev_aesni_mb_create(const char *name,
- struct rte_vdev_device *vdev,
- struct rte_cryptodev_pmd_init_params *init_params)
-{
- struct rte_cryptodev *dev;
- struct aesni_mb_private *internals;
- enum aesni_mb_vector_mode vector_mode;
- MB_MGR *mb_mgr;
-
- dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
- if (dev == NULL) {
- AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
- return -ENODEV;
- }
+ .dev_infos_get = ipsec_mb_info_get,
- /* Check CPU for supported vector instruction set */
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
- vector_mode = RTE_AESNI_MB_AVX512;
- else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
- vector_mode = RTE_AESNI_MB_AVX2;
- else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
- vector_mode = RTE_AESNI_MB_AVX;
- else
- vector_mode = RTE_AESNI_MB_SSE;
+ .queue_pair_setup = ipsec_mb_qp_setup,
+ .queue_pair_release = ipsec_mb_qp_release,
- dev->driver_id = cryptodev_driver_id;
- dev->dev_ops = rte_aesni_mb_pmd_ops;
+ .sym_cpu_process = aesni_mb_process_bulk,
- /* register rx/tx burst functions for data path */
- dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
- dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
-
- dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
- RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
- RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
- RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
- RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
- RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+ .sym_session_get_size = ipsec_mb_sym_session_get_size,
+ .sym_session_configure = ipsec_mb_sym_session_configure,
+ .sym_session_clear = ipsec_mb_sym_session_clear
+};
#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
- struct rte_security_ctx *security_instance;
- security_instance = rte_malloc("aesni_mb_sec",
- sizeof(struct rte_security_ctx),
- RTE_CACHE_LINE_SIZE);
- if (security_instance == NULL) {
- AESNI_MB_LOG(ERR, "rte_security_ctx memory alloc failed");
- rte_cryptodev_pmd_destroy(dev);
- return -ENOMEM;
+/**
+ * Configure a aesni multi-buffer session from a security session
+ * configuration
+ */
+static int
+aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
+ struct rte_security_session *sess,
+ struct rte_mempool *mempool)
+{
+ void *sess_private_data;
+ struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
+ int ret;
+
+ if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
+ conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
+ IPSEC_MB_LOG(ERR, "Invalid security protocol");
+ return -EINVAL;
}
- security_instance->device = (void *)dev;
- security_instance->ops = rte_aesni_mb_pmd_sec_ops;
- security_instance->sess_cnt = 0;
- dev->security_ctx = security_instance;
- dev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
-#endif
+ if (rte_mempool_get(mempool, &sess_private_data)) {
+ IPSEC_MB_LOG(ERR, "Couldn't get object from session mempool");
+ return -ENOMEM;
+ }
- /* Check CPU for support for AES instruction set */
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
- else
- AESNI_MB_LOG(WARNING, "AES instructions not supported by CPU");
+ ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
+ sess_private_data);
- dev->feature_flags |= vec_mode_to_flags(vector_mode);
+ if (ret != 0) {
+ IPSEC_MB_LOG(ERR, "Failed to configure session parameters");
- mb_mgr = alloc_init_mb_mgr(vector_mode);
- if (mb_mgr == NULL) {
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
- rte_free(dev->security_ctx);
- dev->security_ctx = NULL;
-#endif
- rte_cryptodev_pmd_destroy(dev);
- return -ENOMEM;
+ /* Return session to mempool */
+ rte_mempool_put(mempool, sess_private_data);
+ return ret;
}
- /* Set vector instructions mode supported */
- internals = dev->data->dev_private;
+ set_sec_session_private_data(sess, sess_private_data);
- internals->vector_mode = vector_mode;
- internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
- internals->mb_mgr = mb_mgr;
-
- AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
- imb_get_version_str());
- return 0;
+ return ret;
}
+/** Clear the memory of session so it does not leave key material behind */
static int
-cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
+aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
+ struct rte_security_session *sess)
{
- struct rte_cryptodev_pmd_init_params init_params = {
- "",
- sizeof(struct aesni_mb_private),
- rte_socket_id(),
- RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
- };
- const char *name, *args;
- int retval;
-
- name = rte_vdev_device_name(vdev);
- if (name == NULL)
- return -EINVAL;
+ void *sess_priv = get_sec_session_private_data(sess);
- args = rte_vdev_device_args(vdev);
+ if (sess_priv) {
+ struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
- retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
- if (retval) {
- AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
- args);
- return -EINVAL;
+ memset(sess_priv, 0, sizeof(struct aesni_mb_session));
+ set_sec_session_private_data(sess, NULL);
+ rte_mempool_put(sess_mp, sess_priv);
}
-
- return cryptodev_aesni_mb_create(name, vdev, &init_params);
+ return 0;
}
-static int
-cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
+/** Get security capabilities for aesni multi-buffer */
+static const struct rte_security_capability *
+aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
{
- struct rte_cryptodev *cryptodev;
- struct aesni_mb_private *internals;
- const char *name;
+ return aesni_mb_pmd_security_cap;
+}
- name = rte_vdev_device_name(vdev);
- if (name == NULL)
- return -EINVAL;
+static struct rte_security_ops aesni_mb_pmd_sec_ops = {
+ .session_create = aesni_mb_pmd_sec_sess_create,
+ .session_update = NULL,
+ .session_stats_get = NULL,
+ .session_destroy = aesni_mb_pmd_sec_sess_destroy,
+ .set_pkt_metadata = NULL,
+ .capabilities_get = aesni_mb_pmd_sec_capa_get
+};
- cryptodev = rte_cryptodev_pmd_get_named_dev(name);
- if (cryptodev == NULL)
- return -ENODEV;
+struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
- internals = cryptodev->data->dev_private;
+static int
+aesni_mb_configure_dev(struct rte_cryptodev *dev)
+{
+ struct rte_security_ctx *security_instance;
- free_mb_mgr(internals->mb_mgr);
- if (RTE_PER_LCORE(sync_mb_mgr)) {
- free_mb_mgr(RTE_PER_LCORE(sync_mb_mgr));
- RTE_PER_LCORE(sync_mb_mgr) = NULL;
+ security_instance = rte_malloc("aesni_mb_sec",
+ sizeof(struct rte_security_ctx),
+ RTE_CACHE_LINE_SIZE);
+ if (security_instance != NULL) {
+ security_instance->device = (void *)dev;
+ security_instance->ops = rte_aesni_mb_pmd_sec_ops;
+ security_instance->sess_cnt = 0;
+ dev->security_ctx = security_instance;
+
+ return 0;
}
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
- rte_free(cryptodev->security_ctx);
- cryptodev->security_ctx = NULL;
+ return -ENOMEM;
+}
+
#endif
- return rte_cryptodev_pmd_destroy(cryptodev);
+static int
+aesni_mb_probe(struct rte_vdev_device *vdev)
+{
+ return ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_AESNI_MB);
}
static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
- .probe = cryptodev_aesni_mb_probe,
- .remove = cryptodev_aesni_mb_remove
+ .probe = aesni_mb_probe,
+ .remove = ipsec_mb_remove
};
static struct cryptodev_driver aesni_mb_crypto_drv;
-RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
+RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD,
+ cryptodev_aesni_mb_pmd_drv);
RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
- "max_nb_queue_pairs=<int> "
- "socket_id=<int>");
-RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
- cryptodev_aesni_mb_pmd_drv.driver,
- cryptodev_driver_id);
-RTE_LOG_REGISTER_DEFAULT(aesni_mb_logtype_driver, NOTICE);
+ "max_nb_queue_pairs=<int> socket_id=<int>");
+RTE_PMD_REGISTER_CRYPTO_DRIVER(
+ aesni_mb_crypto_drv,
+ cryptodev_aesni_mb_pmd_drv.driver,
+ pmd_driver_id_aesni_mb);
+
+/* Constructor function to register aesni-mb PMD */
+RTE_INIT(ipsec_mb_register_aesni_mb)
+{
+ struct ipsec_mb_internals *aesni_mb_data =
+ &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_AESNI_MB];
+
+ aesni_mb_data->caps = aesni_mb_capabilities;
+ aesni_mb_data->dequeue_burst = aesni_mb_dequeue_burst;
+ aesni_mb_data->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+ RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+ RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
+ RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
+ RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
+ RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+
+ aesni_mb_data->internals_priv_size = 0;
+ aesni_mb_data->ops = &aesni_mb_pmd_ops;
+ aesni_mb_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);
+ aesni_mb_data->queue_pair_configure = NULL;
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+ aesni_mb_data->security_ops = &aesni_mb_pmd_sec_ops;
+ aesni_mb_data->dev_config = aesni_mb_configure_dev;
+ aesni_mb_data->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
+#endif
+ aesni_mb_data->session_configure = aesni_mb_session_configure;
+ aesni_mb_data->session_priv_size = sizeof(struct aesni_mb_session);
+}
similarity index 55%
rename from drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
rename to drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h
@@ -1,19 +1,28 @@
/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2017 Intel Corporation
+ * Copyright(c) 2015-2021 Intel Corporation
*/
-#include <string.h>
+#ifndef _PMD_AESNI_MB_PRIV_H_
+#define _PMD_AESNI_MB_PRIV_H_
-#include <rte_string_fns.h>
-#include <rte_common.h>
-#include <rte_malloc.h>
+#include <intel-ipsec-mb.h>
+
+#if defined(RTE_LIB_SECURITY)
+#define AESNI_MB_DOCSIS_SEC_ENABLED 1
+#include <rte_security.h>
+#include <rte_security_driver.h>
#include <rte_ether.h>
-#include <cryptodev_pmd.h>
+#endif
-#include "aesni_mb_pmd_private.h"
+#include "ipsec_mb_private.h"
+#define AES_CCM_DIGEST_MIN_LEN 4
+#define AES_CCM_DIGEST_MAX_LEN 16
+#define HMAC_MAX_BLOCK_SIZE 128
+#define HMAC_IPAD_VALUE (0x36)
+#define HMAC_OPAD_VALUE (0x5C)
-static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
+static const struct rte_cryptodev_capabilities aesni_mb_capabilities[] = {
{ /* MD5 HMAC */
.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
{.sym = {
@@ -315,13 +324,8 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
.block_size = 16,
.key_size = {
.min = 16,
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
.max = 32,
.increment = 16
-#else
- .max = 16,
- .increment = 0
-#endif
},
.iv_size = {
.min = 16,
@@ -400,13 +404,8 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
.block_size = 16,
.key_size = {
.min = 16,
-#if IMB_VERSION(0, 54, 2) <= IMB_VERSION_NUM
.max = 32,
.increment = 16
-#else
- .max = 16,
- .increment = 0
-#endif
},
.digest_size = {
.min = 4,
@@ -502,7 +501,6 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
}, }
}, }
},
-#if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
{ /* AES ECB */
.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
{.sym = {
@@ -519,8 +517,6 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
}, }
}, }
},
-#endif
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
{ /* ZUC (EIA3) */
.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
{.sym = {
@@ -652,8 +648,6 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
}, }
}, }
},
-#endif
-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
{ /* CHACHA20-POLY1305 */
.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
{.sym = {
@@ -673,7 +667,7 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
},
.aad_size = {
.min = 0,
- .max = 240,
+ .max = 1024,
.increment = 1
},
.iv_size = {
@@ -684,10 +678,244 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
}, }
}, }
},
-#endif
RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
};
+uint8_t pmd_driver_id_aesni_mb;
+
+struct aesni_mb_qp_data {
+ uint8_t temp_digests[IMB_MAX_JOBS][DIGEST_LENGTH_MAX];
+ /* *< Buffers used to store the digest generated
+ * by the driver when verifying a digest provided
+ * by the user (using authentication verify operation)
+ */
+};
+
+/* Maximum length for digest */
+#define DIGEST_LENGTH_MAX 64
+static const unsigned int auth_blocksize[] = {
+ [IMB_AUTH_NULL] = 0,
+ [IMB_AUTH_MD5] = 64,
+ [IMB_AUTH_HMAC_SHA_1] = 64,
+ [IMB_AUTH_HMAC_SHA_224] = 64,
+ [IMB_AUTH_HMAC_SHA_256] = 64,
+ [IMB_AUTH_HMAC_SHA_384] = 128,
+ [IMB_AUTH_HMAC_SHA_512] = 128,
+ [IMB_AUTH_AES_XCBC] = 16,
+ [IMB_AUTH_AES_CCM] = 16,
+ [IMB_AUTH_AES_CMAC] = 16,
+ [IMB_AUTH_AES_GMAC] = 16,
+ [IMB_AUTH_SHA_1] = 64,
+ [IMB_AUTH_SHA_224] = 64,
+ [IMB_AUTH_SHA_256] = 64,
+ [IMB_AUTH_SHA_384] = 128,
+ [IMB_AUTH_SHA_512] = 128,
+ [IMB_AUTH_ZUC_EIA3_BITLEN] = 16,
+ [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 16,
+ [IMB_AUTH_KASUMI_UIA1] = 16
+};
+
+/**
+ * Get the blocksize in bytes for a specified authentication algorithm
+ *
+ * @Note: this function will not return a valid value for a non-valid
+ * authentication algorithm
+ */
+static inline unsigned int
+get_auth_algo_blocksize(IMB_HASH_ALG algo)
+{
+ return auth_blocksize[algo];
+}
+
+static const unsigned int auth_truncated_digest_byte_lengths[] = {
+ [IMB_AUTH_MD5] = 12,
+ [IMB_AUTH_HMAC_SHA_1] = 12,
+ [IMB_AUTH_HMAC_SHA_224] = 14,
+ [IMB_AUTH_HMAC_SHA_256] = 16,
+ [IMB_AUTH_HMAC_SHA_384] = 24,
+ [IMB_AUTH_HMAC_SHA_512] = 32,
+ [IMB_AUTH_AES_XCBC] = 12,
+ [IMB_AUTH_AES_CMAC] = 12,
+ [IMB_AUTH_AES_CCM] = 8,
+ [IMB_AUTH_NULL] = 0,
+ [IMB_AUTH_AES_GMAC] = 12,
+ [IMB_AUTH_SHA_1] = 20,
+ [IMB_AUTH_SHA_224] = 28,
+ [IMB_AUTH_SHA_256] = 32,
+ [IMB_AUTH_SHA_384] = 48,
+ [IMB_AUTH_SHA_512] = 64,
+ [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
+ [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
+ [IMB_AUTH_KASUMI_UIA1] = 4
+};
+
+/**
+ * Get the IPsec specified truncated length in bytes of the HMAC digest for a
+ * specified authentication algorithm
+ *
+ * @Note: this function will not return a valid value for a non-valid
+ * authentication algorithm
+ */
+static inline unsigned int
+get_truncated_digest_byte_length(IMB_HASH_ALG algo)
+{
+ return auth_truncated_digest_byte_lengths[algo];
+}
+
+static const unsigned int auth_digest_byte_lengths[] = {
+ [IMB_AUTH_MD5] = 16,
+ [IMB_AUTH_HMAC_SHA_1] = 20,
+ [IMB_AUTH_HMAC_SHA_224] = 28,
+ [IMB_AUTH_HMAC_SHA_256] = 32,
+ [IMB_AUTH_HMAC_SHA_384] = 48,
+ [IMB_AUTH_HMAC_SHA_512] = 64,
+ [IMB_AUTH_AES_XCBC] = 16,
+ [IMB_AUTH_AES_CMAC] = 16,
+ [IMB_AUTH_AES_CCM] = 16,
+ [IMB_AUTH_AES_GMAC] = 16,
+ [IMB_AUTH_NULL] = 0,
+ [IMB_AUTH_SHA_1] = 20,
+ [IMB_AUTH_SHA_224] = 28,
+ [IMB_AUTH_SHA_256] = 32,
+ [IMB_AUTH_SHA_384] = 48,
+ [IMB_AUTH_SHA_512] = 64,
+ [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
+ [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
+ [IMB_AUTH_KASUMI_UIA1] = 4
+ /**< Vector mode dependent pointer table of the multi-buffer APIs */
+
+};
+
+/**
+ * Get the full digest size in bytes for a specified authentication algorithm
+ * (if available in the Multi-buffer library)
+ *
+ * @Note: this function will not return a valid value for a non-valid
+ * authentication algorithm
+ */
+static inline unsigned int
+get_digest_byte_length(IMB_HASH_ALG algo)
+{
+ return auth_digest_byte_lengths[algo];
+}
+
+/** AES-NI multi-buffer private session structure */
+struct aesni_mb_session {
+ IMB_CIPHER_MODE cipher_mode;
+ IMB_CIPHER_DIRECTION cipher_direction;
+ IMB_HASH_ALG hash_alg;
+ IMB_CHAIN_ORDER chain_order;
+ /* common job fields */
+ struct {
+ uint16_t length;
+ uint16_t offset;
+ } iv;
+ struct {
+ uint16_t length;
+ uint16_t offset;
+ } auth_iv;
+ /* *< IV parameters
+ */
+
+ /* * Cipher Parameters
+ */
+ struct {
+ /* * Cipher direction - encrypt / decrypt */
+ IMB_CIPHER_DIRECTION direction;
+ /* * Cipher mode - CBC / Counter */
+ IMB_CIPHER_MODE mode;
+
+ uint64_t key_length_in_bytes;
+
+ union {
+ struct {
+ uint32_t encode[60] __rte_aligned(16);
+ /* *< encode key */
+ uint32_t decode[60] __rte_aligned(16);
+ /* *< decode key */
+ } expanded_aes_keys;
+ /* *< Expanded AES keys - Allocating space to
+ * contain the maximum expanded key size which
+ * is 240 bytes for 256 bit AES, calculate by:
+ * ((key size (bytes)) *
+ * ((number of rounds) + 1))
+ */
+ struct {
+ const void *ks_ptr[3];
+ uint64_t key[3][16];
+ } exp_3des_keys;
+ /* *< Expanded 3DES keys */
+
+ struct gcm_key_data gcm_key;
+ /* *< Expanded GCM key */
+ uint8_t zuc_cipher_key[16];
+ /* *< ZUC cipher key */
+ snow3g_key_schedule_t pKeySched_snow3g_cipher;
+ /* *< SNOW3G scheduled cipher key */
+ kasumi_key_sched_t pKeySched_kasumi_cipher;
+ /* *< KASUMI scheduled cipher key */
+ };
+ } cipher;
+
+ /* *< Authentication Parameters */
+ struct {
+ IMB_HASH_ALG algo; /* *< Authentication Algorithm */
+ enum rte_crypto_auth_operation operation;
+ /* *< auth operation generate or verify */
+ union {
+ struct {
+ uint8_t inner[128] __rte_aligned(16);
+ /* *< inner pad */
+ uint8_t outer[128] __rte_aligned(16);
+ /* *< outer pad */
+ } pads;
+ /* *< HMAC Authentication pads -
+ * allocating space for the maximum pad
+ * size supported which is 128 bytes for
+ * SHA512
+ */
+
+ struct {
+ uint32_t k1_expanded[44] __rte_aligned(16);
+ /* *< k1 (expanded key). */
+ uint8_t k2[16] __rte_aligned(16);
+ /* *< k2. */
+ uint8_t k3[16] __rte_aligned(16);
+ /* *< k3. */
+ } xcbc;
+
+ struct {
+ uint32_t expkey[60] __rte_aligned(16);
+ /* *< k1 (expanded key). */
+ uint32_t skey1[4] __rte_aligned(16);
+ /* *< k2. */
+ uint32_t skey2[4] __rte_aligned(16);
+ /* *< k3. */
+ } cmac;
+ /* *< Expanded XCBC authentication keys */
+ uint8_t zuc_auth_key[16];
+ /* *< ZUC authentication key */
+ snow3g_key_schedule_t pKeySched_snow3g_auth;
+ /* *< SNOW3G scheduled authentication key */
+ kasumi_key_sched_t pKeySched_kasumi_auth;
+ /* *< KASUMI scheduled authentication key */
+ };
+ /* * Generated digest size by the Multi-buffer library */
+ uint16_t gen_digest_len;
+ /* * Requested digest size from Cryptodev */
+ uint16_t req_digest_len;
+
+ } auth;
+ struct {
+ /* * AAD data length */
+ uint16_t aad_len;
+ } aead;
+} __rte_cache_aligned;
+
+typedef void (*hash_one_block_t)(const void *data, void *digest);
+typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys,
+ void *dec_exp_keys);
+
#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
static const struct rte_cryptodev_capabilities
aesni_mb_pmd_security_crypto_cap[] = {
@@ -738,389 +966,4 @@ static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
};
#endif
-/** Configure device */
-static int
-aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
- __rte_unused struct rte_cryptodev_config *config)
-{
- return 0;
-}
-
-/** Start device */
-static int
-aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
-{
- return 0;
-}
-
-/** Stop device */
-static void
-aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
-{
-}
-
-/** Close device */
-static int
-aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
-{
- return 0;
-}
-
-
-/** Get device statistics */
-static void
-aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
- struct rte_cryptodev_stats *stats)
-{
- int qp_id;
-
- for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
- struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
-
- stats->enqueued_count += qp->stats.enqueued_count;
- stats->dequeued_count += qp->stats.dequeued_count;
-
- stats->enqueue_err_count += qp->stats.enqueue_err_count;
- stats->dequeue_err_count += qp->stats.dequeue_err_count;
- }
-}
-
-/** Reset device statistics */
-static void
-aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
-{
- int qp_id;
-
- for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
- struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
-
- memset(&qp->stats, 0, sizeof(qp->stats));
- }
-}
-
-
-/** Get device info */
-static void
-aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
- struct rte_cryptodev_info *dev_info)
-{
- struct aesni_mb_private *internals = dev->data->dev_private;
-
- if (dev_info != NULL) {
- dev_info->driver_id = dev->driver_id;
- dev_info->feature_flags = dev->feature_flags;
- dev_info->capabilities = aesni_mb_pmd_capabilities;
- dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
- /* No limit of number of sessions */
- dev_info->sym.max_nb_sessions = 0;
- }
-}
-
-/** Release queue pair */
-static int
-aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
-{
- struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
- struct rte_ring *r = NULL;
-
- if (qp != NULL) {
- r = rte_ring_lookup(qp->name);
- if (r)
- rte_ring_free(r);
- if (qp->mb_mgr)
- free_mb_mgr(qp->mb_mgr);
- rte_free(qp);
- dev->data->queue_pairs[qp_id] = NULL;
- }
- return 0;
-}
-
-/** set a unique name for the queue pair based on it's name, dev_id and qp_id */
-static int
-aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
- struct aesni_mb_qp *qp)
-{
- unsigned n = snprintf(qp->name, sizeof(qp->name),
- "aesni_mb_pmd_%u_qp_%u",
- dev->data->dev_id, qp->id);
-
- if (n >= sizeof(qp->name))
- return -1;
-
- return 0;
-}
-
-/** Create a ring to place processed operations on */
-static struct rte_ring *
-aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
- unsigned int ring_size, int socket_id)
-{
- struct rte_ring *r;
- char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
-
- unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
-
- if (n >= sizeof(ring_name))
- return NULL;
-
- r = rte_ring_lookup(ring_name);
- if (r) {
- if (rte_ring_get_size(r) >= ring_size) {
- AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
- ring_name);
- return r;
- }
-
- AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
- ring_name);
- return NULL;
- }
-
- return rte_ring_create(ring_name, ring_size, socket_id,
- RING_F_SP_ENQ | RING_F_SC_DEQ);
-}
-
-/** Setup a queue pair */
-static int
-aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
- const struct rte_cryptodev_qp_conf *qp_conf,
- int socket_id)
-{
- struct aesni_mb_qp *qp = NULL;
- struct aesni_mb_private *internals = dev->data->dev_private;
- int ret = -1;
-
- /* Free memory prior to re-allocation if needed. */
- if (dev->data->queue_pairs[qp_id] != NULL)
- aesni_mb_pmd_qp_release(dev, qp_id);
-
- /* Allocate the queue pair data structure. */
- qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
- RTE_CACHE_LINE_SIZE, socket_id);
- if (qp == NULL)
- return -ENOMEM;
-
- qp->id = qp_id;
- dev->data->queue_pairs[qp_id] = qp;
-
- if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
- goto qp_setup_cleanup;
-
-
- qp->mb_mgr = alloc_mb_mgr(0);
- if (qp->mb_mgr == NULL) {
- ret = -ENOMEM;
- goto qp_setup_cleanup;
- }
-
- switch (internals->vector_mode) {
- case RTE_AESNI_MB_SSE:
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
- init_mb_mgr_sse(qp->mb_mgr);
- break;
- case RTE_AESNI_MB_AVX:
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
- init_mb_mgr_avx(qp->mb_mgr);
- break;
- case RTE_AESNI_MB_AVX2:
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
- init_mb_mgr_avx2(qp->mb_mgr);
- break;
- case RTE_AESNI_MB_AVX512:
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
- init_mb_mgr_avx512(qp->mb_mgr);
- break;
- default:
- AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
- internals->vector_mode);
- goto qp_setup_cleanup;
- }
-
- qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
- qp_conf->nb_descriptors, socket_id);
- if (qp->ingress_queue == NULL) {
- ret = -1;
- goto qp_setup_cleanup;
- }
-
- qp->sess_mp = qp_conf->mp_session;
- qp->sess_mp_priv = qp_conf->mp_session_private;
-
- memset(&qp->stats, 0, sizeof(qp->stats));
-
- char mp_name[RTE_MEMPOOL_NAMESIZE];
-
- snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
- "digest_mp_%u_%u", dev->data->dev_id, qp_id);
- return 0;
-
-qp_setup_cleanup:
- if (qp) {
- if (qp->mb_mgr)
- free_mb_mgr(qp->mb_mgr);
- rte_free(qp);
- }
-
- return ret;
-}
-
-/** Returns the size of the aesni multi-buffer session structure */
-static unsigned
-aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
-{
- return sizeof(struct aesni_mb_session);
-}
-
-/** Configure a aesni multi-buffer session from a crypto xform chain */
-static int
-aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
- struct rte_crypto_sym_xform *xform,
- struct rte_cryptodev_sym_session *sess,
- struct rte_mempool *mempool)
-{
- void *sess_private_data;
- struct aesni_mb_private *internals = dev->data->dev_private;
- int ret;
-
- if (unlikely(sess == NULL)) {
- AESNI_MB_LOG(ERR, "invalid session struct");
- return -EINVAL;
- }
-
- if (rte_mempool_get(mempool, &sess_private_data)) {
- AESNI_MB_LOG(ERR,
- "Couldn't get object from session mempool");
- return -ENOMEM;
- }
-
- ret = aesni_mb_set_session_parameters(internals->mb_mgr,
- sess_private_data, xform);
- if (ret != 0) {
- AESNI_MB_LOG(ERR, "failed configure session parameters");
-
- /* Return session to mempool */
- rte_mempool_put(mempool, sess_private_data);
- return ret;
- }
-
- set_sym_session_private_data(sess, dev->driver_id,
- sess_private_data);
-
- return 0;
-}
-
-/** Clear the memory of session so it doesn't leave key material behind */
-static void
-aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
- struct rte_cryptodev_sym_session *sess)
-{
- uint8_t index = dev->driver_id;
- void *sess_priv = get_sym_session_private_data(sess, index);
-
- /* Zero out the whole structure */
- if (sess_priv) {
- memset(sess_priv, 0, sizeof(struct aesni_mb_session));
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
- set_sym_session_private_data(sess, index, NULL);
- rte_mempool_put(sess_mp, sess_priv);
- }
-}
-
-struct rte_cryptodev_ops aesni_mb_pmd_ops = {
- .dev_configure = aesni_mb_pmd_config,
- .dev_start = aesni_mb_pmd_start,
- .dev_stop = aesni_mb_pmd_stop,
- .dev_close = aesni_mb_pmd_close,
-
- .stats_get = aesni_mb_pmd_stats_get,
- .stats_reset = aesni_mb_pmd_stats_reset,
-
- .dev_infos_get = aesni_mb_pmd_info_get,
-
- .queue_pair_setup = aesni_mb_pmd_qp_setup,
- .queue_pair_release = aesni_mb_pmd_qp_release,
-
- .sym_cpu_process = aesni_mb_cpu_crypto_process_bulk,
-
- .sym_session_get_size = aesni_mb_pmd_sym_session_get_size,
- .sym_session_configure = aesni_mb_pmd_sym_session_configure,
- .sym_session_clear = aesni_mb_pmd_sym_session_clear
-};
-
-struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
-
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
-/**
- * Configure a aesni multi-buffer session from a security session
- * configuration
- */
-static int
-aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
- struct rte_security_session *sess,
- struct rte_mempool *mempool)
-{
- void *sess_private_data;
- struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
- int ret;
-
- if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
- conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
- AESNI_MB_LOG(ERR, "Invalid security protocol");
- return -EINVAL;
- }
-
- if (rte_mempool_get(mempool, &sess_private_data)) {
- AESNI_MB_LOG(ERR, "Couldn't get object from session mempool");
- return -ENOMEM;
- }
-
- ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
- sess_private_data);
-
- if (ret != 0) {
- AESNI_MB_LOG(ERR, "Failed to configure session parameters");
-
- /* Return session to mempool */
- rte_mempool_put(mempool, sess_private_data);
- return ret;
- }
-
- set_sec_session_private_data(sess, sess_private_data);
-
- return ret;
-}
-
-/** Clear the memory of session so it doesn't leave key material behind */
-static int
-aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
- struct rte_security_session *sess)
-{
- void *sess_priv = get_sec_session_private_data(sess);
-
- if (sess_priv) {
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
- memset(sess_priv, 0, sizeof(struct aesni_mb_session));
- set_sec_session_private_data(sess, NULL);
- rte_mempool_put(sess_mp, sess_priv);
- }
- return 0;
-}
-
-/** Get security capabilities for aesni multi-buffer */
-static const struct rte_security_capability *
-aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
-{
- return aesni_mb_pmd_security_cap;
-}
-
-static struct rte_security_ops aesni_mb_pmd_sec_ops = {
- .session_create = aesni_mb_pmd_sec_sess_create,
- .session_update = NULL,
- .session_stats_get = NULL,
- .session_destroy = aesni_mb_pmd_sec_sess_destroy,
- .set_pkt_metadata = NULL,
- .capabilities_get = aesni_mb_pmd_sec_capa_get
-};
-
-struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
-#endif
+#endif /* _PMD_AESNI_MB_PRIV_H_ */
@@ -7,7 +7,6 @@ endif
drivers = [
'aesni_gcm',
- 'aesni_mb',
'armv8',
'bcmfs',
'caam_jr',