From patchwork Mon Mar 16 19:44:34 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mairtin o Loingsigh X-Patchwork-Id: 66735 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E5324A0559; Mon, 16 Mar 2020 20:44:43 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 0669D1C0C0; Mon, 16 Mar 2020 20:44:43 +0100 (CET) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id 3F4FF1AFF for ; Mon, 16 Mar 2020 20:44:40 +0100 (CET) IronPort-SDR: z5lV8ZXplNSiGy9kr0jAc5DuRlUIWM2c6q0WC1F+jLUFLEZCkOLmlfkDTOxYzsh2ls34RC4iyo UyLd32QqmEEg== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 16 Mar 2020 12:44:38 -0700 IronPort-SDR: dNb1OK69h26RDOs0oW5lTBY2b1Vn5VVwQRCbAlN4vSDY0beRNOKlflvtvdSkPloU9lN5cB99+3 hYBw9qnczcPw== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.70,561,1574150400"; d="scan'208";a="279146136" Received: from irvmail001.ir.intel.com ([163.33.26.43]) by fmsmga002.fm.intel.com with ESMTP; 16 Mar 2020 12:44:37 -0700 Received: from sivswdev08.ir.intel.com (sivswdev08.ir.intel.com [10.237.217.47]) by irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id 02GJibJH002798; Mon, 16 Mar 2020 19:44:37 GMT Received: from sivswdev08.ir.intel.com (localhost [127.0.0.1]) by sivswdev08.ir.intel.com with ESMTP id 02GJibuN027386; Mon, 16 Mar 2020 19:44:37 GMT Received: (from moloings@localhost) by sivswdev08.ir.intel.com with LOCAL id 02GJiaT2027382; Mon, 16 Mar 2020 19:44:36 GMT From: Mairtin o Loingsigh To: pablo.de.lara.guarch@intel.com Cc: dev@dpdk.org, Mairtin o Loingsigh Date: Mon, 16 Mar 2020 19:44:34 +0000 Message-Id: <1584387874-27334-1-git-send-email-mairtin.oloingsigh@intel.com> X-Mailer: git-send-email 1.7.0.7 Subject: [dpdk-dev] [PATCH] raw/aesni_mb: introduce aesni_mb rawdev pmd X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Exposes AESNI-MB via rawdev API to allow the submission of operations requiring error detect and other non symmetric crypto transforms. Signed-off-by: Mairtin o Loingsigh --- config/common_base | 5 + drivers/raw/Makefile | 2 + drivers/raw/aesni_mb/Makefile | 43 + drivers/raw/aesni_mb/aesni_mb_rawdev.c | 1215 ++++++++++++++++++++ drivers/raw/aesni_mb/aesni_mb_rawdev_private.h | 121 ++ drivers/raw/aesni_mb/meson.build | 26 + .../aesni_mb/rte_pmd_rawdev_aesni_mb_version.map | 3 + mk/rte.app.mk | 5 + 8 files changed, 1420 insertions(+), 0 deletions(-) create mode 100644 drivers/raw/aesni_mb/Makefile create mode 100644 drivers/raw/aesni_mb/aesni_mb_rawdev.c create mode 100644 drivers/raw/aesni_mb/aesni_mb_rawdev_private.h create mode 100644 drivers/raw/aesni_mb/meson.build create mode 100644 drivers/raw/aesni_mb/rte_pmd_rawdev_aesni_mb_version.map diff --git a/config/common_base b/config/common_base index 7ca2f28..b2dbbb7 100644 --- a/config/common_base +++ b/config/common_base @@ -820,6 +820,11 @@ CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EP_RAWDEV=y CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV=y # +# Compile PMD for AESNI raw device +# +CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV=n + +# # Compile librte_ring # CONFIG_RTE_LIBRTE_RING=y diff --git a/drivers/raw/Makefile b/drivers/raw/Makefile index 80b043e..c00a12a 100644 --- a/drivers/raw/Makefile +++ b/drivers/raw/Makefile @@ -14,5 +14,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += ioat DIRS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV) += octeontx2_dma DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EP_RAWDEV) += octeontx2_ep +DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV) += aesni_mb +DEPDIRS-aesni_mb := common include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/drivers/raw/aesni_mb/Makefile b/drivers/raw/aesni_mb/Makefile new file mode 100644 index 0000000..75c6cad --- /dev/null +++ b/drivers/raw/aesni_mb/Makefile @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2015-2018 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_pmd_aesni_mb_rawdev.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -DALLOW_EXPERIMENTAL_API + +# versioning export map +EXPORT_MAP := rte_pmd_rawdev_aesni_mb_version.map + +# external library dependencies +LDLIBS += -lIPSec_MB +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_rawdev +LDLIBS += -lrte_bus_vdev +LDLIBS += -lrte_multi_fn + +IMB_HDR = $(shell echo '\#include ' | \ + $(CC) -E $(EXTRA_CFLAGS) - | grep 'intel-ipsec-mb.h' | \ + head -n1 | cut -d'"' -f2) + +# Detect library version +IMB_VERSION = $(shell grep -e "IMB_VERSION_STR" $(IMB_HDR) | cut -d'"' -f2) +IMB_VERSION_NUM = $(shell grep -e "IMB_VERSION_NUM" $(IMB_HDR) | cut -d' ' -f3) + +ifeq ($(IMB_VERSION),) +$(error "IPSec_MB version >= 0.53.3 is required") +endif + +ifeq ($(shell expr $(IMB_VERSION_NUM) \< 0x3503), 1) +$(error "IPSec_MB version >= 0.53.3 is required") +endif + +SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV) += aesni_mb_rawdev.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/raw/aesni_mb/aesni_mb_rawdev.c b/drivers/raw/aesni_mb/aesni_mb_rawdev.c new file mode 100644 index 0000000..576ed84 --- /dev/null +++ b/drivers/raw/aesni_mb/aesni_mb_rawdev.c @@ -0,0 +1,1215 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Intel Corporation + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aesni_mb_rawdev_private.h" + +#define AESNI_MB_RAWDEV_PMD_SOCKET_ID_ARG ("socket_id") + +static const char * const rawdev_pmd_valid_params[] = { + AESNI_MB_RAWDEV_PMD_SOCKET_ID_ARG +}; + +static const unsigned err_detect_output_byte_lengths[] = { + [IMB_AUTH_DOCSIS_CRC32] = 4, +}; + +static inline unsigned +get_output_byte_length(JOB_HASH_ALG algo) +{ + return err_detect_output_byte_lengths[algo]; +} + +int +pmd_parse_uint_arg(const char *key __rte_unused, + const char *value, void *extra_args); + +int +parse_input_args(unsigned int *socket_id, + const char *args); + +static int +aesni_mb_rawdev_pmd_config(const struct rte_rawdev *dev, + rte_rawdev_obj_t config) +{ + struct rte_multi_fn_device_info *dev_priv; + struct rte_multi_fn_dev_config *conf; + struct aesni_mb_rawdev_dev *d; + + dev_priv = config; + conf = dev_priv->config; + d = dev->dev_private; + d->num_queue_pair = conf->nb_queues; + + d->queue_pairs = rte_zmalloc_socket("aesni_rawdev_qps", + d->num_queue_pair * sizeof(struct aesni_mb_rawdev_qp), + RTE_CACHE_LINE_SIZE, dev->socket_id); + + if (!d->queue_pairs) { + AESNI_MB_RAWDEV_LOG(ERR, "Unable to allocate queue pairs"); + return -ENOMEM; + } + + return 0; +} + +static int +aesni_mb_rawdev_pmd_start(__rte_unused struct rte_rawdev *dev) +{ + return 0; +} + +static void +aesni_mb_rawdev_pmd_stop(__rte_unused struct rte_rawdev *dev) +{ +} + +static int +aesni_mb_rawdev_pmd_close(__rte_unused struct rte_rawdev *dev) +{ + return 0; +} + +static int +set_session_err_detect_parameters(struct aesni_mb_rawdev_session *sess, + struct rte_multi_fn_err_detect_xform *xform) +{ + + if (xform == NULL) { + AESNI_MB_RAWDEV_LOG(ERR, "Invalid error detection xform"); + return -EINVAL; + } + + /* Select error detect generate/verify */ + if (xform->op == RTE_MULTI_FN_ERR_DETECT_OP_VERIFY) + sess->err_detect.operation = + RTE_MULTI_FN_ERR_DETECT_OP_VERIFY; + else if (xform->op == RTE_MULTI_FN_ERR_DETECT_OP_GENERATE) + sess->err_detect.operation = + RTE_MULTI_FN_ERR_DETECT_OP_GENERATE; + else { + AESNI_MB_RAWDEV_LOG(ERR, "Unsupported err_detect operation"); + return -ENOTSUP; + } + + if (xform->algo == RTE_MULTI_FN_ERR_DETECT_CRC32_ETH) { + sess->err_detect.algo = IMB_AUTH_DOCSIS_CRC32; + } else { + AESNI_MB_RAWDEV_LOG(ERR, + "Unsupported error detect algorithm selection"); + return -ENOTSUP; + } + + sess->err_detect.gen_output_len = + get_output_byte_length(sess->err_detect.algo); + + return 0; +} + +static int +set_session_cipher_parameters(const MB_MGR *mb_mgr, + struct aesni_mb_rawdev_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + + if (xform == NULL) { + sess->cipher.mode = IMB_CIPHER_NULL; + return -EINVAL; + } + + if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) { + AESNI_MB_RAWDEV_LOG(ERR, "Crypto xform not of type cipher"); + return -EINVAL; + } + + /* Select cipher direction */ + switch (xform->cipher.op) { + case RTE_CRYPTO_CIPHER_OP_ENCRYPT: + sess->cipher.direction = IMB_DIR_ENCRYPT; + break; + case RTE_CRYPTO_CIPHER_OP_DECRYPT: + sess->cipher.direction = IMB_DIR_DECRYPT; + break; + default: + AESNI_MB_RAWDEV_LOG(ERR, "Invalid cipher operation parameter"); + return -EINVAL; + } + + if (xform->cipher.algo == RTE_CRYPTO_CIPHER_AES_DOCSISBPI) { + sess->cipher.mode = IMB_CIPHER_DOCSIS_SEC_BPI; + } else { + AESNI_MB_RAWDEV_LOG(ERR, "Unsupported cipher mode parameter"); + return -ENOTSUP; + } + + /* Set IV parameters */ + sess->iv.offset = xform->cipher.iv.offset; + sess->iv.length = xform->cipher.iv.length; + + /* Check key length and choose key expansion function for AES */ + switch (xform->cipher.key.length) { + case IMB_KEY_AES_128_BYTES: + sess->cipher.key_length_in_bytes = IMB_KEY_AES_128_BYTES; + IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + break; + case IMB_KEY_AES_256_BYTES: + sess->cipher.key_length_in_bytes = IMB_KEY_AES_256_BYTES; + IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + break; + default: + AESNI_MB_RAWDEV_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + + return 0; +} + +static int +aesni_mb_rawdev_pmd_enq(struct rte_rawdev *dev, struct rte_rawdev_buf **ops, + unsigned int nb_ops, rte_rawdev_obj_t q_id) +{ + unsigned int nb_enqueued; + struct aesni_mb_rawdev_dev *hw = dev->dev_private; + struct aesni_mb_rawdev_qp *qp; + + qp = hw->queue_pairs[*(uint16_t *)q_id]; + + nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue, + (void **)ops, nb_ops, NULL); + + qp->stats.enqueued_count += nb_enqueued; + + return nb_enqueued; +} + +static inline struct aesni_mb_rawdev_session * +get_session(struct rte_multi_fn_op *op) +{ + struct aesni_mb_rawdev_session *sess = NULL; + + if (likely(op->sess != NULL)) + sess = op->sess->sess_private_data; + else + op->op_status = RTE_MULTI_FN_STATUS_INVALID_SESSION; + + return sess; +} + +static inline uint64_t +err_start_offset(struct rte_multi_fn_op *op, + struct aesni_mb_rawdev_session *session, uint32_t oop) +{ + struct rte_mbuf *m_src, *m_dst; + uint8_t *p_src, *p_dst; + uintptr_t u_src, u_dst; + uint32_t cipher_end, err_detect_end; + struct rte_crypto_sym_op *s_op; + struct rte_multi_fn_err_detect_op *e_op; + + s_op = &op->next->crypto_sym; + e_op = &op->err_detect; + m_src = op->m_src; + m_dst = op->m_dst; + + /* Only cipher then error detect needs special calculation. */ + if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH) + return op->next->err_detect.data.offset; + + p_src = rte_pktmbuf_mtod(m_src, uint8_t *); + p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); + u_src = (uintptr_t)p_src; + u_dst = (uintptr_t)p_dst + e_op->data.offset; + + /** + * Copy the content between cipher offset and err detect offset + * for generating correct output. + */ + if (s_op->cipher.data.offset > e_op->data.offset) + memcpy(p_dst + e_op->data.offset, + p_src + e_op->data.offset, + s_op->cipher.data.offset - + e_op->data.offset); + + /** + * Copy the content between (cipher offset + length) and + * (error detect offset + length) for generating correct output + */ + cipher_end = s_op->cipher.data.offset + s_op->cipher.data.length; + err_detect_end = e_op->data.offset + e_op->data.length; + if (cipher_end < err_detect_end) + memcpy(p_dst + cipher_end, p_src + cipher_end, + err_detect_end - cipher_end); + + /** + * Since intel-ipsec-mb only supports positive values, + * we need to deduct the correct offset between src and dst. + */ + + return u_src < u_dst ? (u_dst - u_src) : + (UINT64_MAX - u_src + u_dst + 1); +} + +static inline int +set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_rawdev_qp *qp, + struct rte_multi_fn_op *op, uint8_t *output_idx) +{ + struct rte_mbuf *m_src, *m_dst; + struct rte_crypto_sym_op *s_op; + struct rte_multi_fn_err_detect_op *e_op; + uint32_t m_offset, oop; + struct aesni_mb_rawdev_session *session; + + session = get_session(op); + if (session == NULL) { + op->op_status = RTE_MULTI_FN_STATUS_INVALID_SESSION; + return -1; + } + + if (op->next == NULL) { + op->op_status = RTE_MULTI_FN_OP_STATUS_FAILURE; + return -1; + } + + m_src = op->m_src; + + /* Set crypto operation */ + job->chain_order = session->chain_order; + if (session->chain_order == IMB_ORDER_CIPHER_HASH) { + s_op = &op->crypto_sym; + e_op = &op->next->err_detect; + } else { + s_op = &op->next->crypto_sym; + e_op = &op->err_detect; + } + + /* Set cipher parameters */ + job->cipher_direction = session->cipher.direction; + job->cipher_mode = session->cipher.mode; + + job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes; + + /* Set authentication parameters */ + job->hash_alg = session->err_detect.algo; + + job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode; + + if (!op->m_dst) { + /* in-place operation */ + m_dst = m_src; + oop = 0; + } else if (op->m_dst == op->m_src) { + /* in-place operation */ + m_dst = m_src; + oop = 0; + } else { + /* out-of-place operation */ + m_dst = op->m_dst; + oop = 1; + } + + m_offset = s_op->cipher.data.offset; + + /* Set output location */ + if (session->err_detect.operation == + RTE_MULTI_FN_ERR_DETECT_OP_VERIFY) { + job->auth_tag_output = qp->temp_digests[*output_idx]; + *output_idx = (*output_idx + 1) % MAX_JOBS; + } else { + job->auth_tag_output = e_op->output.data; + } + + /* Set output length */ + job->auth_tag_output_len_in_bytes = session->err_detect.gen_output_len; + + /* Set IV parameters */ + job->iv_len_in_bytes = session->iv.length; + + /* Data Parameters */ + job->src = rte_pktmbuf_mtod(m_src, uint8_t *); + job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset); + + job->cipher_start_src_offset_in_bytes = s_op->cipher.data.offset; + job->msg_len_to_cipher_in_bytes = s_op->cipher.data.length; + + job->hash_start_src_offset_in_bytes = err_start_offset(op, + session, oop); + job->msg_len_to_hash_in_bytes = e_op->data.length; + + job->iv = (uint8_t *)op + session->iv.offset; + + job->user_data = op; + + return 0; +} + +static inline void +verify_crc(JOB_AES_HMAC *job, uint8_t *status) +{ + uint16_t crc_offset = job->hash_start_src_offset_in_bytes + + job->msg_len_to_hash_in_bytes - + job->cipher_start_src_offset_in_bytes; + uint8_t *embedded_crc = job->dst + crc_offset; + + if (!job->msg_len_to_hash_in_bytes) + return; + + /* Verify CRC (at the end of the message) */ + if (memcmp(job->auth_tag_output, embedded_crc, 4) != 0) + *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; +} + +static inline struct rte_multi_fn_op * +post_process_mb_job(JOB_AES_HMAC *job) +{ + struct rte_multi_fn_op *op = (struct rte_multi_fn_op *) + job->user_data; + struct aesni_mb_rawdev_session *sess = op->sess->sess_private_data; + + AESNI_MB_RAWDEV_LOG(INFO, "struct rte_multi_fn_session %p", + op->sess); + AESNI_MB_RAWDEV_LOG(INFO, "struct aesni_mb_rawdev_session %p", sess); + + if (likely(op->op_status == + RTE_MULTI_FN_OP_STATUS_NOT_PROCESSED)) { + switch (job->status) { + case STS_COMPLETED: + op->op_status = RTE_MULTI_FN_OP_STATUS_SUCCESS; + + if (job->hash_alg == IMB_AUTH_NULL) + break; + + if (sess->err_detect.operation == + RTE_MULTI_FN_ERR_DETECT_OP_VERIFY) + verify_crc(job, &op->op_status); + break; + default: + op->op_status = RTE_MULTI_FN_OP_STATUS_FAILURE; + } + } + + return op; +} + +static unsigned +handle_completed_jobs(struct aesni_mb_rawdev_qp *qp, JOB_AES_HMAC *job, + struct rte_multi_fn_op **ops, uint16_t nb_ops) +{ + struct rte_multi_fn_op *op = NULL; + unsigned processed_jobs = 0; + + while (job != NULL) { + op = post_process_mb_job(job); + + if (op) { + ops[processed_jobs++] = op; + qp->stats.dequeued_count++; + } else { + qp->stats.dequeue_err_count++; + break; + } + if (processed_jobs == nb_ops) + break; + + job = IMB_GET_COMPLETED_JOB(qp->mb_mgr); + } + + return processed_jobs; +} + +static inline uint16_t +flush_mb_mgr(struct aesni_mb_rawdev_qp *qp, struct rte_multi_fn_op **ops, + uint16_t nb_ops) +{ + int processed_ops = 0; + + /* Flush the remaining jobs */ + JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr); + + AESNI_MB_RAWDEV_LOG(INFO, "qp->mb_mgr %p", qp->mb_mgr); + + if (job) + processed_ops += handle_completed_jobs(qp, job, + &ops[processed_ops], nb_ops - processed_ops); + + return processed_ops; +} + +static inline JOB_AES_HMAC * +set_job_null_op(JOB_AES_HMAC *job, struct rte_multi_fn_op *op) +{ + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_NULL; + job->cipher_direction = IMB_DIR_DECRYPT; + + /* Set user data to be crypto operation data struct */ + job->user_data = op; + + return job; +} + +static int +aesni_mb_rawdev_pmd_deq(struct rte_rawdev *dev, struct rte_rawdev_buf **ops, + unsigned int nb_ops, rte_rawdev_obj_t q_id) +{ + struct aesni_mb_rawdev_qp *qp; + struct rte_multi_fn_op *op; + JOB_AES_HMAC *job; + struct aesni_mb_rawdev_dev *hw = dev->dev_private; + + qp = hw->queue_pairs[*(uint16_t *)q_id]; + int retval; + unsigned int processed_jobs = 0; + + if (unlikely(nb_ops == 0)) + return 0; + + AESNI_MB_RAWDEV_LOG(INFO, "qp->mb_mgr %p", qp->mb_mgr); + + uint8_t output_idx = qp->output_idx; + do { + /* Get next free mb job struct from mb manager */ + job = IMB_GET_NEXT_JOB(qp->mb_mgr); + if (unlikely(job == NULL)) { + /* if no free mb job structs we need to flush mb_mgr */ + processed_jobs += flush_mb_mgr(qp, + (struct rte_multi_fn_op **) + &ops[processed_jobs], + nb_ops - processed_jobs); + + if (nb_ops == processed_jobs) + break; + + job = IMB_GET_NEXT_JOB(qp->mb_mgr); + } + + /* + * Get next operation to process from ingress queue. + * There is no need to return the job to the MB_MGR + * if there are no more operations to process, since the MB_MGR + * can use that pointer again in next get_next calls. + */ + retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op); + if (retval < 0) + break; + + retval = set_mb_job_params(job, qp, op, &output_idx); + if (unlikely(retval != 0)) { + qp->stats.dequeue_err_count++; + set_job_null_op(job, op); + } + + /* Submit job to multi-buffer for processing */ +#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG + job = IMB_SUBMIT_JOB(qp->mb_mgr); +#else + job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr); +#endif + /* + * If submit returns a processed job then handle it, + * before submitting subsequent jobs + */ + if (job) + processed_jobs += handle_completed_jobs(qp, job, + (struct rte_multi_fn_op **) + &ops[processed_jobs], + nb_ops - processed_jobs); + + } while (processed_jobs < nb_ops); + + qp->output_idx = output_idx; + + if (processed_jobs < 1) + processed_jobs += flush_mb_mgr(qp, + (struct rte_multi_fn_op **) + &ops[processed_jobs], + nb_ops - processed_jobs); + + return processed_jobs; +} + +static int +aesni_mb_rawdev_pmd_qp_release(struct rte_rawdev *dev, uint16_t qp_id) +{ + struct aesni_mb_rawdev_dev *hw = dev->dev_private; + struct aesni_mb_rawdev_qp *qp = hw->queue_pairs[qp_id]; + struct rte_ring *r = NULL; + + if (qp != NULL) { + r = rte_ring_lookup(qp->name); + if (r) + rte_ring_free(r); + if (qp->mb_mgr) + free_mb_mgr(qp->mb_mgr); + rte_free(qp); + hw->queue_pairs[qp_id] = NULL; + } + return 0; +} + +static int +qp_set_unique_name(struct rte_rawdev *dev, + struct aesni_mb_rawdev_qp *qp) +{ + unsigned n = snprintf(qp->name, sizeof(qp->name), + "aesni_mb_pmd_%u_qp_%u", + dev->dev_id, qp->id); + + if (n >= sizeof(qp->name)) + return -1; + + return 0; +} + +static struct rte_ring * +qp_create_processed_ops_ring(struct aesni_mb_rawdev_qp *qp, + unsigned int ring_size, int socket_id) +{ + struct rte_ring *r; + char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name)); + + if (n >= sizeof(ring_name)) + return NULL; + + r = rte_ring_lookup(ring_name); + if (r) { + if (rte_ring_get_size(r) >= ring_size) { + AESNI_MB_RAWDEV_LOG(INFO, "Reusing existing ring %s for processed ops", + ring_name); + return r; + } + + AESNI_MB_RAWDEV_LOG(ERR, "Unable to reuse existing ring %s for processed ops", + ring_name); + return NULL; + } + + return rte_ring_create(ring_name, ring_size, socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ); +} + +static int +aesni_mb_rawdev_pmd_qp_setup(struct rte_rawdev *dev, uint16_t qp_id, + rte_rawdev_obj_t qp_c) +{ + + struct aesni_mb_rawdev_qp *qp = NULL; + struct aesni_mb_rawdev_dev *hw = dev->dev_private; + struct aesni_mb_rawdev_private *internals = &hw->priv; + const struct rte_multi_fn_qp_config *qp_conf = + (const struct rte_multi_fn_qp_config *)qp_c; + int ret = -1; + + /* Free memory prior to re-allocation if needed. */ + if (hw->queue_pairs[qp_id] != NULL) + aesni_mb_rawdev_pmd_qp_release(dev, qp_id); + + /* Allocate the queue pair data structure. */ + qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp), + RTE_CACHE_LINE_SIZE, + internals->socket_id); + if (qp == NULL) + return -ENOMEM; + + qp->id = qp_id; + hw->queue_pairs[qp_id] = qp; + + if (qp_set_unique_name(dev, qp)) + goto qp_setup_cleanup; + + qp->mb_mgr = alloc_mb_mgr(0); + AESNI_MB_RAWDEV_LOG(INFO, "mb_mgr %p", qp->mb_mgr); + if (qp->mb_mgr == NULL) { + ret = -ENOMEM; + goto qp_setup_cleanup; + } + + switch (internals->vector_mode) { + case AESNI_MB_RAWDEV_SSE: + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + init_mb_mgr_sse(qp->mb_mgr); + break; + case AESNI_MB_RAWDEV_AVX: + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + init_mb_mgr_avx(qp->mb_mgr); + break; + case AESNI_MB_RAWDEV_AVX2: + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx2(qp->mb_mgr); + break; + case AESNI_MB_RAWDEV_AVX512: + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512; + init_mb_mgr_avx512(qp->mb_mgr); + break; + default: + AESNI_MB_RAWDEV_LOG(ERR, "Unsupported vector mode %u\n", + internals->vector_mode); + goto qp_setup_cleanup; + } + + qp->ingress_queue = qp_create_processed_ops_ring(qp, + qp_conf->nb_descriptors, rte_socket_id()); + if (qp->ingress_queue == NULL) { + ret = -1; + goto qp_setup_cleanup; + } + + memset(&qp->stats, 0, sizeof(qp->stats)); + + char mp_name[RTE_MEMPOOL_NAMESIZE]; + + snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, + "digest_mp_%u_%u", dev->dev_id, qp_id); + return 0; + +qp_setup_cleanup: + if (qp) { + if (qp->mb_mgr) + free_mb_mgr(qp->mb_mgr); + rte_free(qp); + } + + return ret; +} + +static uint16_t +aesni_mb_rawdev_pmd_qp_count(struct rte_rawdev *dev) +{ + struct aesni_mb_rawdev_dev *hw = dev->dev_private; + return hw->num_queue_pair; +} + +static enum aesni_mb_rawdev_op +session_support(struct rte_multi_fn_xform *xform) +{ + struct rte_crypto_sym_xform *crypto_sym; + struct rte_multi_fn_err_detect_xform *err_detect; + struct rte_multi_fn_xform *next; + enum aesni_mb_rawdev_op op = AESNI_MB_RAWDEV_OP_NOT_SUPPORTED; + + next = xform->next; + /* err detect generate -> cipher encrypt */ + if (xform->type == RTE_MULTI_FN_XFORM_TYPE_ERR_DETECT) { + + err_detect = &xform->err_detect; + if ((err_detect->algo == + RTE_MULTI_FN_ERR_DETECT_CRC32_ETH) && + (err_detect->op == + RTE_MULTI_FN_ERR_DETECT_OP_GENERATE) && + next != NULL && + next->type == RTE_MULTI_FN_XFORM_TYPE_CRYPTO_SYM) { + + crypto_sym = &next->crypto_sym; + if (crypto_sym->type == + RTE_CRYPTO_SYM_XFORM_CIPHER && + crypto_sym->cipher.op == + RTE_CRYPTO_CIPHER_OP_ENCRYPT && + crypto_sym->cipher.algo == + RTE_CRYPTO_CIPHER_AES_DOCSISBPI && + crypto_sym->cipher.key.length == + IMB_KEY_AES_128_BYTES && + crypto_sym->cipher.iv.length == + AES_BLOCK_SIZE) { + op = AESNI_MB_RAWDEV_OP_ERR_DETECT_CIPHER; + } + } + /* cipher decrypt -> err detect verify */ + } else if (xform->type == RTE_MULTI_FN_XFORM_TYPE_CRYPTO_SYM) { + crypto_sym = &xform->crypto_sym; + if (crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT && + crypto_sym->cipher.algo == + RTE_CRYPTO_CIPHER_AES_DOCSISBPI && + crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES && + crypto_sym->cipher.iv.length == AES_BLOCK_SIZE && + next != NULL && + next->type == RTE_MULTI_FN_XFORM_TYPE_ERR_DETECT) { + + err_detect = &next->err_detect; + if ((err_detect->algo == + RTE_MULTI_FN_ERR_DETECT_CRC32_ETH) && + (err_detect->op == + RTE_MULTI_FN_ERR_DETECT_OP_VERIFY)) { + op = AESNI_MB_RAWDEV_OP_CIPHER_ERR_DETECT; + } + } + + } + + return op; +} + +static struct rte_multi_fn_session * +aesni_mb_pmd_session_create(struct rte_rawdev *dev, + struct rte_multi_fn_xform *xform, + int socket_id) +{ + int ret; + enum aesni_mb_rawdev_op op; + struct rte_multi_fn_err_detect_xform *err_detect_xform; + struct rte_crypto_sym_xform *cipher_xform; + struct rte_multi_fn_session *session; + struct aesni_mb_rawdev_session *aesni_sess = NULL; + struct aesni_mb_rawdev_dev *d = dev->dev_private; + struct aesni_mb_rawdev_private *internals = &d->priv; + + op = session_support(xform); + + /* Alloc rte_multi_fn_session */ + session = rte_zmalloc_socket("rte_multi_fn_session", + sizeof(struct rte_multi_fn_session), + RTE_CACHE_LINE_MIN_SIZE, + socket_id); + + if (!session) { + AESNI_MB_RAWDEV_LOG(ERR, "rte_multi_fn_session alloc failed"); + return NULL; + } + + /* Alloc aesni_mb_rawdev_session */ + aesni_sess = rte_zmalloc_socket("aesni_mb_rawdev_session", + sizeof(struct aesni_mb_rawdev_session), + RTE_CACHE_LINE_MIN_SIZE, + socket_id); + + if (!aesni_sess) { + AESNI_MB_RAWDEV_LOG(ERR, "aesni_mb_rawdev_session alloc failed"); + return NULL; + } + + session->sess_private_data = aesni_sess; + + switch (op) { + case AESNI_MB_RAWDEV_OP_ERR_DETECT_CIPHER: + aesni_sess->chain_order = IMB_ORDER_HASH_CIPHER; + err_detect_xform = &xform->err_detect; + cipher_xform = &xform->next->crypto_sym; + break; + case AESNI_MB_RAWDEV_OP_CIPHER_ERR_DETECT: + aesni_sess->chain_order = IMB_ORDER_CIPHER_HASH; + cipher_xform = &xform->crypto_sym; + err_detect_xform = &xform->next->err_detect; + break; + default: + AESNI_MB_RAWDEV_LOG(ERR, "Unsupported multi-function xform chain"); + return NULL; + } + + /* Default IV length = 0 */ + aesni_sess->iv.length = 0; + + ret = set_session_err_detect_parameters(aesni_sess, + err_detect_xform); + + if (ret != 0) { + AESNI_MB_RAWDEV_LOG(ERR, "Invalid/unsupported error detect parameters"); + return NULL; + } + + ret = set_session_cipher_parameters(internals->mb_mgr, + aesni_sess, + cipher_xform); + + if (ret != 0) { + AESNI_MB_RAWDEV_LOG(ERR, "Invalid/unsupported cipher parameters"); + return NULL; + } + + return session; +} + +static int +aesni_mb_rawdev_pmd_session_destroy(struct rte_rawdev *dev __rte_unused, + struct rte_multi_fn_session *sess) +{ + + if (sess) { + if (sess->sess_private_data) + rte_free(sess->sess_private_data); + rte_free(sess); + } + + return 0; +} + +#define MAX_QUEUES 64 + +static void +aesni_mb_rawdev_pmd_info_get(struct rte_rawdev *dev __rte_unused, + rte_rawdev_obj_t dev_info) +{ + struct rte_multi_fn_dev_config *config = NULL; + struct rte_multi_fn_device_info *dev_priv = dev_info; + dev_priv->create = aesni_mb_pmd_session_create; + dev_priv->destroy = aesni_mb_rawdev_pmd_session_destroy; + config = dev_priv->config; + config->nb_queues = MAX_QUEUES; +} + +int +pmd_parse_uint_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + int i; + char *end; + errno = 0; + + i = strtol(value, &end, 10); + if (*end != 0 || errno != 0 || i < 0) + return -EINVAL; + + *((uint32_t *)extra_args) = i; + return 0; +} + +int +parse_input_args(unsigned int *socket_id, + const char *args) +{ + struct rte_kvargs *kvlist = NULL; + int ret = 0; + + if (socket_id == NULL) + return -EINVAL; + + if (args) { + kvlist = rte_kvargs_parse(args, rawdev_pmd_valid_params); + if (kvlist == NULL) + return -EINVAL; + + ret = rte_kvargs_process(kvlist, + AESNI_MB_RAWDEV_PMD_SOCKET_ID_ARG, + &pmd_parse_uint_arg, + socket_id); + if (ret < 0) + goto free_kvlist; + + } + +free_kvlist: + rte_kvargs_free(kvlist); + return ret; +} + +#define RTE_RAWDEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 8 + +static const char * const xstat_names[] = { + "successful_enqueues", "successful_dequeues", + "failed_enqueues", "failed_dequeues", +}; + +static int +aesni_mb_rawdev_xstats_get(const struct rte_rawdev *dev, + const unsigned int ids[], + uint64_t values[], unsigned int n) +{ + unsigned int i; + struct aesni_mb_rawdev_dev *hw = dev->dev_private; + struct aesni_mb_rawdev_qp *qp; + struct aesni_mb_rawdev_stats stats = {0}; + int qp_id; + + for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) { + qp = hw->queue_pairs[qp_id]; + + stats.enqueued_count += qp->stats.enqueued_count; + stats.dequeued_count += qp->stats.dequeued_count; + + stats.enqueue_err_count += qp->stats.enqueue_err_count; + stats.dequeue_err_count += qp->stats.dequeue_err_count; + } + + for (i = 0; i < n; i++) { + switch (ids[i]) { + case 0: + values[i] = stats.enqueued_count; + break; + case 1: + values[i] = stats.dequeued_count; + break; + case 2: + values[i] = stats.enqueue_err_count; + break; + case 3: + values[i] = stats.dequeue_err_count; + break; + default: + values[i] = 0; + break; + } + } + return n; +} + +static int +aesni_mb_rawdev_xstats_get_names(const struct rte_rawdev *dev, + struct rte_rawdev_xstats_name *names, + unsigned int size) +{ + unsigned int i; + + RTE_SET_USED(dev); + if (size < RTE_DIM(xstat_names)) + return RTE_DIM(xstat_names); + + for (i = 0; i < RTE_DIM(xstat_names); i++) + strlcpy(names[i].name, xstat_names[i], sizeof(names[i])); + + return RTE_DIM(xstat_names); +} + +static int +aesni_mb_rawdev_xstats_reset(struct rte_rawdev *dev, const uint32_t *ids, + uint32_t nb_ids) +{ + struct aesni_mb_rawdev_dev *hw = dev->dev_private; + struct aesni_mb_rawdev_qp *qp; + unsigned int i; + int qp_id; + + if (!ids) { + + for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) { + qp = hw->queue_pairs[qp_id]; + qp->stats.enqueued_count = 0; + qp->stats.dequeued_count = 0; + qp->stats.enqueue_err_count = 0; + qp->stats.dequeue_err_count = 0; + } + return 0; + } + + for (i = 0; i < nb_ids; i++) { + switch (ids[i]) { + case 0: + for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) { + qp = hw->queue_pairs[qp_id]; + qp->stats.enqueued_count = 0; + } + break; + case 1: + for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) { + qp = hw->queue_pairs[qp_id]; + qp->stats.dequeued_count = 0; + } + break; + case 2: + for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) { + qp = hw->queue_pairs[qp_id]; + qp->stats.enqueue_err_count = 0; + } + break; + case 3: + for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) { + qp = hw->queue_pairs[qp_id]; + qp->stats.dequeue_err_count = 0; + } + break; + default: + AESNI_MB_RAWDEV_LOG(ERR, + "Invalid xstat id - cannot reset val"); + break; + } + } + + return 0; +} + +static int +rawdev_aesni_mb_create(struct rte_rawdev *dev, + struct rte_multi_fn_dev_config *init_params) +{ + + static const struct rte_rawdev_ops aesni_mb_raw_ops = { + .enqueue_bufs = aesni_mb_rawdev_pmd_enq, + .dequeue_bufs = aesni_mb_rawdev_pmd_deq, + .dev_configure = aesni_mb_rawdev_pmd_config, + .dev_info_get = aesni_mb_rawdev_pmd_info_get, + .dev_start = aesni_mb_rawdev_pmd_start, + .dev_stop = aesni_mb_rawdev_pmd_stop, + .dev_close = aesni_mb_rawdev_pmd_close, + .queue_setup = aesni_mb_rawdev_pmd_qp_setup, + .queue_release = aesni_mb_rawdev_pmd_qp_release, + .queue_count = aesni_mb_rawdev_pmd_qp_count, + .xstats_get = aesni_mb_rawdev_xstats_get, + .xstats_get_names = aesni_mb_rawdev_xstats_get_names, + .xstats_reset = aesni_mb_rawdev_xstats_reset, + }; + + struct aesni_mb_rawdev_dev *hw; + struct aesni_mb_rawdev_private *internals; + enum aesni_mb_rawdev_vector_mode vector_mode; + MB_MGR *mb_mgr; + + hw = dev->dev_private; + + /* Check CPU for supported vector instruction set */ + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) + vector_mode = AESNI_MB_RAWDEV_AVX512; + else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) + vector_mode = AESNI_MB_RAWDEV_AVX2; + else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) + vector_mode = AESNI_MB_RAWDEV_AVX; + else + vector_mode = AESNI_MB_RAWDEV_SSE; + + dev->dev_ops = &aesni_mb_raw_ops; + + hw->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + + /* Check CPU for support for AES instruction set */ + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI; + else + AESNI_MB_RAWDEV_LOG(WARNING, "AES instructions not supported by CPU"); + + mb_mgr = alloc_mb_mgr(0); + AESNI_MB_RAWDEV_LOG(INFO, "create() mb_mgr %p", mb_mgr); + + if (mb_mgr == NULL) + return -ENOMEM; + + switch (vector_mode) { + case AESNI_MB_RAWDEV_SSE: + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + init_mb_mgr_sse(mb_mgr); + break; + case AESNI_MB_RAWDEV_AVX: + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + init_mb_mgr_avx(mb_mgr); + break; + case AESNI_MB_RAWDEV_AVX2: + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx2(mb_mgr); + break; + case AESNI_MB_RAWDEV_AVX512: + hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512; + init_mb_mgr_avx512(mb_mgr); + break; + default: + AESNI_MB_RAWDEV_LOG(ERR, "Unsupported vector mode %u\n", + vector_mode); + goto error_exit; + } + + /* Set vector instructions mode supported */ + internals = &hw->priv; + + internals->vector_mode = vector_mode; + internals->max_nb_queue_pairs = init_params->nb_queues; + internals->mb_mgr = mb_mgr; + internals->socket_id = init_params->socket_id; + + AESNI_MB_RAWDEV_LOG(INFO, "IPSec Multi-buffer library version used: %s\n", + imb_get_version_str()); + + return 0; + +error_exit: + if (mb_mgr) + free_mb_mgr(mb_mgr); + + rte_rawdev_pmd_release(dev); + + return -1; +} + +static int +aesni_mb_rawdev_probe(struct rte_vdev_device *vdev) +{ + const char *vdev_name; + const char *args; + struct rte_rawdev *rawdev; + struct rte_multi_fn_dev_config init_params = { + .nb_queues = 64, + .socket_id = SOCKET_ID_ANY}; + + vdev_name = rte_vdev_device_name(vdev); + if (vdev_name == NULL) + return -EINVAL; + + args = rte_vdev_device_args(vdev); + + /* Parse args */ + parse_input_args(&init_params.socket_id, args); + + AESNI_MB_RAWDEV_LOG(INFO, "device name %s", vdev_name); + + /* Allocate device structure */ + rawdev = rte_rawdev_pmd_allocate(vdev_name, + sizeof(struct aesni_mb_rawdev_dev), + rte_socket_id()); + + if (!rawdev) { + AESNI_MB_RAWDEV_LOG(ERR, "Unable to allocate raw device"); + return -EINVAL; + } + + rawdev->driver_name = RTE_STR(rawdev_aesni_mb); + + AESNI_MB_RAWDEV_LOG(INFO, "Driver name %s", rawdev->driver_name); + + return rawdev_aesni_mb_create(rawdev, &init_params); +} + +static int +aesni_mb_rawdev_remove(struct rte_vdev_device *vdev) +{ + struct rte_rawdev *dev; + struct aesni_mb_rawdev_dev *hw; + struct aesni_mb_rawdev_private *internals; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) { + AESNI_MB_RAWDEV_LOG(INFO, "Driver name %s", name); + return -EINVAL; + } + + dev = rte_rawdev_pmd_get_named_dev(name); + if (dev == NULL) + return -ENODEV; + + hw = dev->dev_private; + internals = &hw->priv; + + free_mb_mgr(internals->mb_mgr); + + return rte_rawdev_pmd_release(dev); +} + +static struct rte_vdev_driver rawdev_aesni_mb_pmd_drv = { + .probe = aesni_mb_rawdev_probe, + .remove = aesni_mb_rawdev_remove +}; + +RTE_PMD_REGISTER_VDEV(rawdev_aesni_mb, rawdev_aesni_mb_pmd_drv); + +RTE_INIT(aesni_mb_raw_init_log) +{ + aesni_mb_rawdev_logtype_driver = rte_log_register("pmd.raw.aesni_mb"); +} diff --git a/drivers/raw/aesni_mb/aesni_mb_rawdev_private.h b/drivers/raw/aesni_mb/aesni_mb_rawdev_private.h new file mode 100644 index 0000000..12a3f16 --- /dev/null +++ b/drivers/raw/aesni_mb/aesni_mb_rawdev_private.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Intel Corporation + */ + +#ifndef _AESNI_MB_RAWDEV_PRIVATE_H_ +#define _AESNI_MB_RAWDEV_PRIVATE_H_ + +#include +#include +#include + +enum aesni_mb_rawdev_vector_mode { + AESNI_MB_RAWDEV_NOT_SUPPORTED = 0, + AESNI_MB_RAWDEV_SSE, + AESNI_MB_RAWDEV_AVX, + AESNI_MB_RAWDEV_AVX2, + AESNI_MB_RAWDEV_AVX512 +}; + +#define AESNI_MB_RAWDEV_PMD_SOCKET_ID_ARG ("socket_id") + +/* AESNI_MB PMD LOGTYPE DRIVER */ +int aesni_mb_rawdev_logtype_driver; + +#define AESNI_MB_RAWDEV_LOG(level, fmt, ...) \ + rte_log(RTE_LOG_ ## level, aesni_mb_rawdev_logtype_driver, \ + "%s() line %u: " fmt "\n", __func__, __LINE__, \ + ## __VA_ARGS__) + +/* Maximum length for digest */ +#define DIGEST_LENGTH_MAX 4 + +/* AESNI-MB operation order */ +enum aesni_mb_rawdev_op { + AESNI_MB_RAWDEV_OP_ERR_DETECT_CIPHER, + AESNI_MB_RAWDEV_OP_CIPHER_ERR_DETECT, + AESNI_MB_RAWDEV_OP_NOT_SUPPORTED +}; + +static const unsigned auth_digest_byte_lengths[] = { + [IMB_AUTH_DOCSIS_CRC32] = 4, + /**< Vector mode dependent pointer table of the multi-buffer APIs */ +}; + +/** + * Get the full digest size in bytes for a specified authentication algorithm + * (if available in the Multi-buffer library) + * + * @Note: this function will not return a valid value for a non-valid + * authentication algorithm + */ +static inline unsigned +get_digest_byte_length(JOB_HASH_ALG algo) +{ + return auth_digest_byte_lengths[algo]; +} + +/* AESNI-MB device statistics */ +struct aesni_mb_rawdev_stats { + uint64_t enqueued_count; + uint64_t dequeued_count; + uint64_t enqueue_err_count; + uint64_t dequeue_err_count; +}; + +/* Private data structure for each virtual AESNI-MB device */ +struct aesni_mb_rawdev_private { + enum aesni_mb_rawdev_vector_mode vector_mode; + unsigned max_nb_queue_pairs; + MB_MGR *mb_mgr; + unsigned int socket_id; +}; + +/* AESNI-MB queue pair */ +struct aesni_mb_rawdev_qp { + uint16_t id; + char name[RTE_RAWDEV_NAME_MAX_LEN]; + MB_MGR *mb_mgr; + struct rte_ring *ingress_queue; + struct aesni_mb_rawdev_stats stats; + uint8_t output_idx; + uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX]; +} __rte_cache_aligned; + +/* AESNI-MB device data */ +struct aesni_mb_rawdev_dev { + struct aesni_mb_rawdev_private priv; + uint16_t num_queue_pair; + struct aesni_mb_rawdev_qp **queue_pairs; + uint64_t feature_flags; +}; + +/* AESNI-MB private session structure */ +struct aesni_mb_rawdev_session { + JOB_CHAIN_ORDER chain_order; + struct { + uint16_t length; + uint16_t offset; + } iv; + struct { + JOB_CIPHER_DIRECTION direction; + JOB_CIPHER_MODE mode; + + uint64_t key_length_in_bytes; + + union { + struct { + uint32_t encode[60] __rte_aligned(16); + uint32_t decode[60] __rte_aligned(16); + } expanded_aes_keys; + }; + } cipher; + struct { + JOB_HASH_ALG algo; + enum rte_multi_fn_err_detect_operation operation; + uint16_t gen_output_len; + + } err_detect; +} __rte_cache_aligned; + +#endif /* _AESNI_MB_RAWDEV_PRIVATE_H_ */ diff --git a/drivers/raw/aesni_mb/meson.build b/drivers/raw/aesni_mb/meson.build new file mode 100644 index 0000000..3e16874 --- /dev/null +++ b/drivers/raw/aesni_mb/meson.build @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +IMB_required_ver = '0.52.0' +lib = cc.find_library('IPSec_MB', required: false) +if not lib.found() + build = false + reason = 'missing dependency, "libIPSec_MB"' +else + ext_deps += lib + + # version comes with quotes, so we split based on " and take the middle + imb_ver = cc.get_define('IMB_VERSION_STR', + prefix : '#include').split('"')[1] + + if (imb_ver == '') or (imb_ver.version_compare('<' + IMB_required_ver)) + reason = 'IPSec_MB version >= @0@ is required, found version @1@'.format( + IMB_required_ver, imb_ver) + build = false + endif + +endif + +sources = files('rte_aesni_mb_pmd.c', 'rte_aesni_mb_pmd_ops.c') +allow_experimental_apis = true +deps += ['bus_vdev'] diff --git a/drivers/raw/aesni_mb/rte_pmd_rawdev_aesni_mb_version.map b/drivers/raw/aesni_mb/rte_pmd_rawdev_aesni_mb_version.map new file mode 100644 index 0000000..fa9e17c --- /dev/null +++ b/drivers/raw/aesni_mb/rte_pmd_rawdev_aesni_mb_version.map @@ -0,0 +1,3 @@ +DPDK_20.0 { + local: *; +}; \ No newline at end of file diff --git a/mk/rte.app.mk b/mk/rte.app.mk index d295ca0..6740cc0 100644 --- a/mk/rte.app.mk +++ b/mk/rte.app.mk @@ -347,6 +347,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += -lrte_rawdev_ioat _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += -lrte_rawdev_ntb _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV) += -lrte_rawdev_octeontx2_dma _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EP_RAWDEV) += -lrte_rawdev_octeontx2_ep +ifeq ($(CONFIG_RTE_LIBRTE_PMD_AESNI_MB),y) +ifeq ($(CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV) += -lrte_pmd_aesni_mb_rawdev +endif # CONFIG_RTE_LIBRTE_PMD_AESNI_MB +endif # CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV endif # CONFIG_RTE_LIBRTE_RAWDEV endif # !CONFIG_RTE_BUILD_SHARED_LIBS