Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/94776/?format=api
{ "id": 94776, "url": "https://patches.dpdk.org/api/patches/94776/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/0686a7c3fb3a22e37378a8545bc37bce04f4c391.1624481225.git.sthotton@marvell.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<0686a7c3fb3a22e37378a8545bc37bce04f4c391.1624481225.git.sthotton@marvell.com>", "list_archive_url": "https://inbox.dpdk.org/dev/0686a7c3fb3a22e37378a8545bc37bce04f4c391.1624481225.git.sthotton@marvell.com", "date": "2021-06-23T20:53:50", "name": "[v2,2/2] drivers: add octeontx crypto adapter data path", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "2f6d0b29a132f76cdf26ac53edce62d009053b84", "submitter": { "id": 2049, "url": "https://patches.dpdk.org/api/people/2049/?format=api", "name": "Shijith Thotton", "email": "sthotton@marvell.com" }, "delegate": { "id": 6690, "url": "https://patches.dpdk.org/api/users/6690/?format=api", "username": "akhil", "first_name": "akhil", "last_name": "goyal", "email": "gakhil@marvell.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/0686a7c3fb3a22e37378a8545bc37bce04f4c391.1624481225.git.sthotton@marvell.com/mbox/", "series": [ { "id": 17464, "url": "https://patches.dpdk.org/api/series/17464/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=17464", "date": "2021-06-23T20:53:48", "name": "OCTEONTX crypto adapter support", "version": 2, "mbox": "https://patches.dpdk.org/series/17464/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/94776/comments/", "check": "warning", "checks": "https://patches.dpdk.org/api/patches/94776/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 5CFD2A0C41;\n\tWed, 23 Jun 2021 22:54:27 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id D97D141149;\n\tWed, 23 Jun 2021 22:54:20 +0200 (CEST)", "from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com\n [67.231.156.173])\n by mails.dpdk.org (Postfix) with ESMTP id 012A24114F\n for <dev@dpdk.org>; Wed, 23 Jun 2021 22:54:18 +0200 (CEST)", "from pps.filterd (m0045851.ppops.net [127.0.0.1])\n by mx0b-0016f401.pphosted.com (8.16.0.43/8.16.0.43) with SMTP id\n 15NKoPjo007495; Wed, 23 Jun 2021 13:54:18 -0700", "from dc5-exch01.marvell.com ([199.233.59.181])\n by mx0b-0016f401.pphosted.com with ESMTP id 39bptj4y97-1\n (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT);\n Wed, 23 Jun 2021 13:54:18 -0700", "from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH01.marvell.com\n (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.18;\n Wed, 23 Jun 2021 13:54:16 -0700", "from maili.marvell.com (10.69.176.80) by DC5-EXCH02.marvell.com\n (10.69.176.39) with Microsoft SMTP Server id 15.0.1497.18 via Frontend\n Transport; Wed, 23 Jun 2021 13:54:16 -0700", "from localhost.localdomain (unknown [10.28.34.29])\n by maili.marvell.com (Postfix) with ESMTP id 9E5F55B6940;\n Wed, 23 Jun 2021 13:54:13 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com;\n h=from : to : cc :\n subject : date : message-id : in-reply-to : references : mime-version :\n content-transfer-encoding : content-type; s=pfpt0220;\n bh=9ayOgArscm9U3XmlVI1RsVbAgsTgX/C3tH/BS33D4lA=;\n b=jJE1vT+uiVIIQnzi5fScFCh2ayMRygxL3L3p+GVo9pexwiw2DRBPoJO6e/CJc3+EzBb/\n 6x4/WvB0kqcgNnTb8ey3tCZcDzR4mIsGGdZSLi5EWHZ0KKU98FAgkWfq3wyFVjYu0NNf\n /w7MZa2H0qdU/J+kpjQsF/VjE325mg5O0Fn8/QCl9NOui9WUoegjfk0WKyevcjOMTz4I\n JN+dMjHjf1h535uYtkh4QsW/AMjBvNcXTDwW2gdOCQsFo3dWJ4tNkRU+WIv5rWdIvuiJ\n Ld8NVqflZYoivGxqvCOyfYc4hTnehi4Yzu8YnGq7UdWsTc9gzD+KLO8arTvrYuMAWcl8 jg==", "From": "Shijith Thotton <sthotton@marvell.com>", "To": "<dev@dpdk.org>", "CC": "Shijith Thotton <sthotton@marvell.com>, <pbhagavatula@marvell.com>,\n <anoobj@marvell.com>, <jerinj@marvell.com>,\n <abhinandan.gujjar@intel.com>, <adwivedi@marvell.com>, <gakhil@marvell.com>", "Date": "Thu, 24 Jun 2021 02:23:50 +0530", "Message-ID": "\n <0686a7c3fb3a22e37378a8545bc37bce04f4c391.1624481225.git.sthotton@marvell.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<cover.1624481225.git.sthotton@marvell.com>", "References": "<cover.1624379833.git.sthotton@marvell.com>\n <cover.1624481225.git.sthotton@marvell.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Content-Type": "text/plain", "X-Proofpoint-ORIG-GUID": "S9r0xrKH15TSfF1VFM9jrXJgDJTPrwHV", "X-Proofpoint-GUID": "S9r0xrKH15TSfF1VFM9jrXJgDJTPrwHV", "X-Proofpoint-Virus-Version": "vendor=fsecure engine=2.50.10434:6.0.391, 18.0.790\n definitions=2021-06-23_12:2021-06-23,\n 2021-06-23 signatures=0", "Subject": "[dpdk-dev] [PATCH v2 2/2] drivers: add octeontx crypto adapter data\n path", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Added support for crypto adapter OP_FORWARD mode.\n\nAs OcteonTx CPT crypto completions could be out of order, each crypto op\nis enqueued to CPT, dequeued from CPT and enqueued to SSO one-by-one.\n\nSigned-off-by: Shijith Thotton <sthotton@marvell.com>\n---\n doc/guides/rel_notes/release_21_08.rst | 4 +\n drivers/common/cpt/cpt_common.h | 2 +-\n drivers/crypto/octeontx/meson.build | 5 +\n drivers/crypto/octeontx/otx_cryptodev_ops.c | 268 +++++++++++++-----\n drivers/crypto/octeontx/otx_cryptodev_ops.h | 8 +\n drivers/crypto/octeontx/version.map | 9 +\n drivers/event/octeontx/ssovf_evdev.c | 3 +-\n drivers/event/octeontx/ssovf_worker.c | 11 +\n drivers/event/octeontx/ssovf_worker.h | 25 +-\n .../octeontx2/otx2_evdev_crypto_adptr_rx.h | 6 +-\n 10 files changed, 255 insertions(+), 86 deletions(-)", "diff": "diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst\nindex a6ecfdf3ce..70a91cc654 100644\n--- a/doc/guides/rel_notes/release_21_08.rst\n+++ b/doc/guides/rel_notes/release_21_08.rst\n@@ -55,6 +55,10 @@ New Features\n Also, make sure to start the actual text at the margin.\n =======================================================\n \n+* **Updated Marvell OCTEON TX PMD.**\n+\n+ Added support for crypto adapter OP_FORWARD mode.\n+\n \n Removed Items\n -------------\ndiff --git a/drivers/common/cpt/cpt_common.h b/drivers/common/cpt/cpt_common.h\nindex 7fea0ca879..724e5ec736 100644\n--- a/drivers/common/cpt/cpt_common.h\n+++ b/drivers/common/cpt/cpt_common.h\n@@ -54,7 +54,7 @@ struct cpt_request_info {\n \t\tuint64_t ei2;\n \t} ist;\n \tuint8_t *rptr;\n-\tconst struct otx2_cpt_qp *qp;\n+\tconst void *qp;\n \n \t/** Control path fields */\n \tuint64_t time_out;\ndiff --git a/drivers/crypto/octeontx/meson.build b/drivers/crypto/octeontx/meson.build\nindex 37603c5c89..3ae6729e8f 100644\n--- a/drivers/crypto/octeontx/meson.build\n+++ b/drivers/crypto/octeontx/meson.build\n@@ -6,6 +6,7 @@ if not is_linux\n endif\n \n deps += ['bus_pci']\n+deps += ['bus_vdev']\n deps += ['common_cpt']\n deps += ['eventdev']\n \n@@ -18,3 +19,7 @@ sources = files(\n )\n \n includes += include_directories('../../common/cpt')\n+includes += include_directories('../../common/octeontx')\n+includes += include_directories('../../event/octeontx')\n+includes += include_directories('../../mempool/octeontx')\n+includes += include_directories('../../net/octeontx')\ndiff --git a/drivers/crypto/octeontx/otx_cryptodev_ops.c b/drivers/crypto/octeontx/otx_cryptodev_ops.c\nindex d75f4b5f81..2fe04ebdd6 100644\n--- a/drivers/crypto/octeontx/otx_cryptodev_ops.c\n+++ b/drivers/crypto/octeontx/otx_cryptodev_ops.c\n@@ -6,6 +6,8 @@\n #include <rte_bus_pci.h>\n #include <rte_cryptodev.h>\n #include <rte_cryptodev_pmd.h>\n+#include <rte_eventdev.h>\n+#include <rte_event_crypto_adapter.h>\n #include <rte_errno.h>\n #include <rte_malloc.h>\n #include <rte_mempool.h>\n@@ -21,6 +23,8 @@\n #include \"cpt_ucode.h\"\n #include \"cpt_ucode_asym.h\"\n \n+#include \"ssovf_worker.h\"\n+\n static uint64_t otx_fpm_iova[CPT_EC_ID_PMAX];\n \n /* Forward declarations */\n@@ -412,15 +416,17 @@ otx_cpt_asym_session_clear(struct rte_cryptodev *dev,\n \trte_mempool_put(sess_mp, priv);\n }\n \n-static __rte_always_inline int32_t __rte_hot\n+static __rte_always_inline void * __rte_hot\n otx_cpt_request_enqueue(struct cpt_instance *instance,\n \t\t\tstruct pending_queue *pqueue,\n \t\t\tvoid *req, uint64_t cpt_inst_w7)\n {\n \tstruct cpt_request_info *user_req = (struct cpt_request_info *)req;\n \n-\tif (unlikely(pqueue->pending_count >= DEFAULT_CMD_QLEN))\n-\t\treturn -EAGAIN;\n+\tif (unlikely(pqueue->pending_count >= DEFAULT_CMD_QLEN)) {\n+\t\trte_errno = EAGAIN;\n+\t\treturn NULL;\n+\t}\n \n \tfill_cpt_inst(instance, req, cpt_inst_w7);\n \n@@ -434,18 +440,12 @@ otx_cpt_request_enqueue(struct cpt_instance *instance,\n \t/* Default mode of software queue */\n \tmark_cpt_inst(instance);\n \n-\tpqueue->req_queue[pqueue->enq_tail] = (uintptr_t)user_req;\n-\n-\t/* We will use soft queue length here to limit requests */\n-\tMOD_INC(pqueue->enq_tail, DEFAULT_CMD_QLEN);\n-\tpqueue->pending_count += 1;\n-\n \tCPT_LOG_DP_DEBUG(\"Submitted NB cmd with request: %p \"\n \t\t\t \"op: %p\", user_req, user_req->op);\n-\treturn 0;\n+\treturn req;\n }\n \n-static __rte_always_inline int __rte_hot\n+static __rte_always_inline void * __rte_hot\n otx_cpt_enq_single_asym(struct cpt_instance *instance,\n \t\t\tstruct rte_crypto_op *op,\n \t\t\tstruct pending_queue *pqueue)\n@@ -456,11 +456,13 @@ otx_cpt_enq_single_asym(struct cpt_instance *instance,\n \tstruct cpt_asym_sess_misc *sess;\n \tuintptr_t *cop;\n \tvoid *mdata;\n+\tvoid *req;\n \tint ret;\n \n \tif (unlikely(rte_mempool_get(minfo->pool, &mdata) < 0)) {\n \t\tCPT_LOG_DP_ERR(\"Could not allocate meta buffer for request\");\n-\t\treturn -ENOMEM;\n+\t\trte_errno = ENOMEM;\n+\t\treturn NULL;\n \t}\n \n \tsess = get_asym_session_private_data(asym_op->session,\n@@ -506,27 +508,26 @@ otx_cpt_enq_single_asym(struct cpt_instance *instance,\n \n \tdefault:\n \t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;\n-\t\tret = -EINVAL;\n+\t\trte_errno = EINVAL;\n \t\tgoto req_fail;\n \t}\n \n-\tret = otx_cpt_request_enqueue(instance, pqueue, params.req,\n+\treq = otx_cpt_request_enqueue(instance, pqueue, params.req,\n \t\t\t\t sess->cpt_inst_w7);\n-\n-\tif (unlikely(ret)) {\n+\tif (unlikely(req == NULL)) {\n \t\tCPT_LOG_DP_ERR(\"Could not enqueue crypto req\");\n \t\tgoto req_fail;\n \t}\n \n-\treturn 0;\n+\treturn req;\n \n req_fail:\n \tfree_op_meta(mdata, minfo->pool);\n \n-\treturn ret;\n+\treturn NULL;\n }\n \n-static __rte_always_inline int __rte_hot\n+static __rte_always_inline void * __rte_hot\n otx_cpt_enq_single_sym(struct cpt_instance *instance,\n \t\t struct rte_crypto_op *op,\n \t\t struct pending_queue *pqueue)\n@@ -536,6 +537,7 @@ otx_cpt_enq_single_sym(struct cpt_instance *instance,\n \tstruct cpt_request_info *prep_req;\n \tvoid *mdata = NULL;\n \tint ret = 0;\n+\tvoid *req;\n \tuint64_t cpt_op;\n \n \tsess = (struct cpt_sess_misc *)\n@@ -554,23 +556,20 @@ otx_cpt_enq_single_sym(struct cpt_instance *instance,\n \tif (unlikely(ret)) {\n \t\tCPT_LOG_DP_ERR(\"prep cryto req : op %p, cpt_op 0x%x \"\n \t\t\t \"ret 0x%x\", op, (unsigned int)cpt_op, ret);\n-\t\treturn ret;\n+\t\treturn NULL;\n \t}\n \n \t/* Enqueue prepared instruction to h/w */\n-\tret = otx_cpt_request_enqueue(instance, pqueue, prep_req,\n+\treq = otx_cpt_request_enqueue(instance, pqueue, prep_req,\n \t\t\t\t sess->cpt_inst_w7);\n-\n-\tif (unlikely(ret)) {\n+\tif (unlikely(req == NULL))\n \t\t/* Buffer allocated for request preparation need to be freed */\n \t\tfree_op_meta(mdata, instance->meta_info.pool);\n-\t\treturn ret;\n-\t}\n \n-\treturn 0;\n+\treturn req;\n }\n \n-static __rte_always_inline int __rte_hot\n+static __rte_always_inline void * __rte_hot\n otx_cpt_enq_single_sym_sessless(struct cpt_instance *instance,\n \t\t\t\tstruct rte_crypto_op *op,\n \t\t\t\tstruct pending_queue *pend_q)\n@@ -578,12 +577,15 @@ otx_cpt_enq_single_sym_sessless(struct cpt_instance *instance,\n \tconst int driver_id = otx_cryptodev_driver_id;\n \tstruct rte_crypto_sym_op *sym_op = op->sym;\n \tstruct rte_cryptodev_sym_session *sess;\n+\tvoid *req;\n \tint ret;\n \n \t/* Create temporary session */\n \tsess = rte_cryptodev_sym_session_create(instance->sess_mp);\n-\tif (sess == NULL)\n-\t\treturn -ENOMEM;\n+\tif (sess == NULL) {\n+\t\trte_errno = ENOMEM;\n+\t\treturn NULL;\n+\t}\n \n \tret = sym_session_configure(driver_id, sym_op->xform, sess,\n \t\t\t\t instance->sess_mp_priv);\n@@ -592,24 +594,24 @@ otx_cpt_enq_single_sym_sessless(struct cpt_instance *instance,\n \n \tsym_op->session = sess;\n \n-\tret = otx_cpt_enq_single_sym(instance, op, pend_q);\n+\treq = otx_cpt_enq_single_sym(instance, op, pend_q);\n \n-\tif (unlikely(ret))\n+\tif (unlikely(req == NULL))\n \t\tgoto priv_put;\n \n-\treturn 0;\n+\treturn req;\n \n priv_put:\n \tsym_session_clear(driver_id, sess);\n sess_put:\n \trte_mempool_put(instance->sess_mp, sess);\n-\treturn ret;\n+\treturn NULL;\n }\n \n #define OP_TYPE_SYM\t\t0\n #define OP_TYPE_ASYM\t\t1\n \n-static __rte_always_inline int __rte_hot\n+static __rte_always_inline void *__rte_hot\n otx_cpt_enq_single(struct cpt_instance *inst,\n \t\t struct rte_crypto_op *op,\n \t\t struct pending_queue *pqueue,\n@@ -631,7 +633,8 @@ otx_cpt_enq_single(struct cpt_instance *inst,\n \t}\n \n \t/* Should not reach here */\n-\treturn -ENOTSUP;\n+\trte_errno = ENOTSUP;\n+\treturn NULL;\n }\n \n static __rte_always_inline uint16_t __rte_hot\n@@ -640,7 +643,7 @@ otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops,\n {\n \tstruct cpt_instance *instance = (struct cpt_instance *)qptr;\n \tuint16_t count;\n-\tint ret;\n+\tvoid *req;\n \tstruct cpt_vf *cptvf = (struct cpt_vf *)instance;\n \tstruct pending_queue *pqueue = &cptvf->pqueue;\n \n@@ -652,10 +655,14 @@ otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops,\n \twhile (likely(count < nb_ops)) {\n \n \t\t/* Enqueue single op */\n-\t\tret = otx_cpt_enq_single(instance, ops[count], pqueue, op_type);\n+\t\treq = otx_cpt_enq_single(instance, ops[count], pqueue, op_type);\n \n-\t\tif (unlikely(ret))\n+\t\tif (unlikely(req == NULL))\n \t\t\tbreak;\n+\n+\t\tpqueue->req_queue[pqueue->enq_tail] = (uintptr_t)req;\n+\t\tMOD_INC(pqueue->enq_tail, DEFAULT_CMD_QLEN);\n+\t\tpqueue->pending_count += 1;\n \t\tcount++;\n \t}\n \totx_cpt_ring_dbell(instance, count);\n@@ -674,6 +681,80 @@ otx_cpt_enqueue_sym(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)\n \treturn otx_cpt_pkt_enqueue(qptr, ops, nb_ops, OP_TYPE_SYM);\n }\n \n+static __rte_always_inline void\n+submit_request_to_sso(struct ssows *ws, uintptr_t req,\n+\t\t struct rte_event *rsp_info)\n+{\n+\tuint64_t add_work;\n+\n+\tadd_work = rsp_info->flow_id | (RTE_EVENT_TYPE_CRYPTODEV << 28) |\n+\t\t ((uint64_t)(rsp_info->sched_type) << 32);\n+\n+\tif (!rsp_info->sched_type)\n+\t\tssows_head_wait(ws);\n+\n+\trte_atomic_thread_fence(__ATOMIC_RELEASE);\n+\tssovf_store_pair(add_work, req, ws->grps[rsp_info->queue_id]);\n+}\n+\n+static inline union rte_event_crypto_metadata *\n+get_event_crypto_mdata(struct rte_crypto_op *op)\n+{\n+\tunion rte_event_crypto_metadata *ec_mdata;\n+\n+\tif (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\tec_mdata = rte_cryptodev_sym_session_get_user_data(\n+\t\t\t\t\t\t\t op->sym->session);\n+\telse if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&\n+\t\t op->private_data_offset)\n+\t\tec_mdata = (union rte_event_crypto_metadata *)\n+\t\t\t((uint8_t *)op + op->private_data_offset);\n+\telse\n+\t\treturn NULL;\n+\n+\treturn ec_mdata;\n+}\n+\n+uint16_t __rte_hot\n+otx_crypto_adapter_enqueue(void *port, struct rte_crypto_op *op)\n+{\n+\tunion rte_event_crypto_metadata *ec_mdata;\n+\tstruct cpt_instance *instance;\n+\tstruct cpt_request_info *req;\n+\tstruct rte_event *rsp_info;\n+\tuint8_t op_type, cdev_id;\n+\tuint16_t qp_id;\n+\n+\tec_mdata = get_event_crypto_mdata(op);\n+\tif (unlikely(ec_mdata == NULL)) {\n+\t\trte_errno = EINVAL;\n+\t\treturn 0;\n+\t}\n+\n+\tcdev_id = ec_mdata->request_info.cdev_id;\n+\tqp_id = ec_mdata->request_info.queue_pair_id;\n+\trsp_info = &ec_mdata->response_info;\n+\tinstance = rte_cryptodevs[cdev_id].data->queue_pairs[qp_id];\n+\n+\tif (unlikely(!instance->ca_enabled)) {\n+\t\trte_errno = EINVAL;\n+\t\treturn 0;\n+\t}\n+\n+\top_type = op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC ? OP_TYPE_SYM :\n+\t\t\t\t\t\t\t OP_TYPE_ASYM;\n+\treq = otx_cpt_enq_single(instance, op,\n+\t\t\t\t &((struct cpt_vf *)instance)->pqueue, op_type);\n+\tif (unlikely(req == NULL))\n+\t\treturn 0;\n+\n+\totx_cpt_ring_dbell(instance, 1);\n+\treq->qp = instance;\n+\tsubmit_request_to_sso(port, (uintptr_t)req, rsp_info);\n+\n+\treturn 1;\n+}\n+\n static inline void\n otx_cpt_asym_rsa_op(struct rte_crypto_op *cop, struct cpt_request_info *req,\n \t\t struct rte_crypto_rsa_xform *rsa_ctx)\n@@ -820,6 +901,50 @@ otx_cpt_dequeue_post_process(struct rte_crypto_op *cop, uintptr_t *rsp,\n \treturn;\n }\n \n+static inline void\n+free_sym_session_data(const struct cpt_instance *instance,\n+\t\t struct rte_crypto_op *cop)\n+{\n+\tvoid *sess_private_data_t = get_sym_session_private_data(\n+\t\tcop->sym->session, otx_cryptodev_driver_id);\n+\tmemset(sess_private_data_t, 0, cpt_get_session_size());\n+\tmemset(cop->sym->session, 0,\n+\t rte_cryptodev_sym_get_existing_header_session_size(\n+\t\t cop->sym->session));\n+\trte_mempool_put(instance->sess_mp_priv, sess_private_data_t);\n+\trte_mempool_put(instance->sess_mp, cop->sym->session);\n+\tcop->sym->session = NULL;\n+}\n+\n+static __rte_always_inline struct rte_crypto_op *\n+otx_cpt_process_response(const struct cpt_instance *instance, uintptr_t *rsp,\n+\t\t\t uint8_t cc, const uint8_t op_type)\n+{\n+\tstruct rte_crypto_op *cop;\n+\tvoid *metabuf;\n+\n+\tmetabuf = (void *)rsp[0];\n+\tcop = (void *)rsp[1];\n+\n+\t/* Check completion code */\n+\tif (likely(cc == 0)) {\n+\t\t/* H/w success pkt. Post process */\n+\t\totx_cpt_dequeue_post_process(cop, rsp, op_type);\n+\t} else if (cc == ERR_GC_ICV_MISCOMPARE) {\n+\t\t/* auth data mismatch */\n+\t\tcop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n+\t} else {\n+\t\t/* Error */\n+\t\tcop->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t}\n+\n+\tif (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS))\n+\t\tfree_sym_session_data(instance, cop);\n+\tfree_op_meta(metabuf, instance->meta_info.pool);\n+\n+\treturn cop;\n+}\n+\n static __rte_always_inline uint16_t __rte_hot\n otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops,\n \t\t const uint8_t op_type)\n@@ -832,9 +957,6 @@ otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops,\n \tuint8_t ret;\n \tint nb_completed;\n \tstruct pending_queue *pqueue = &cptvf->pqueue;\n-\tstruct rte_crypto_op *cop;\n-\tvoid *metabuf;\n-\tuintptr_t *rsp;\n \n \tpcount = pqueue->pending_count;\n \tcount = (nb_ops > pcount) ? pcount : nb_ops;\n@@ -869,45 +991,11 @@ otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops,\n \tnb_completed = i;\n \n \tfor (i = 0; i < nb_completed; i++) {\n-\n-\t\trsp = (void *)ops[i];\n-\n \t\tif (likely((i + 1) < nb_completed))\n \t\t\trte_prefetch0(ops[i+1]);\n \n-\t\tmetabuf = (void *)rsp[0];\n-\t\tcop = (void *)rsp[1];\n-\n-\t\tops[i] = cop;\n-\n-\t\t/* Check completion code */\n-\n-\t\tif (likely(cc[i] == 0)) {\n-\t\t\t/* H/w success pkt. Post process */\n-\t\t\totx_cpt_dequeue_post_process(cop, rsp, op_type);\n-\t\t} else if (cc[i] == ERR_GC_ICV_MISCOMPARE) {\n-\t\t\t/* auth data mismatch */\n-\t\t\tcop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n-\t\t} else {\n-\t\t\t/* Error */\n-\t\t\tcop->status = RTE_CRYPTO_OP_STATUS_ERROR;\n-\t\t}\n-\n-\t\tif (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {\n-\t\t\tvoid *sess_private_data_t =\n-\t\t\t\tget_sym_session_private_data(cop->sym->session,\n-\t\t\t\t\t\totx_cryptodev_driver_id);\n-\t\t\tmemset(sess_private_data_t, 0,\n-\t\t\t\t\tcpt_get_session_size());\n-\t\t\tmemset(cop->sym->session, 0,\n-\t\t\trte_cryptodev_sym_get_existing_header_session_size(\n-\t\t\t\t\tcop->sym->session));\n-\t\t\trte_mempool_put(instance->sess_mp_priv,\n-\t\t\t\t\tsess_private_data_t);\n-\t\t\trte_mempool_put(instance->sess_mp, cop->sym->session);\n-\t\t\tcop->sym->session = NULL;\n-\t\t}\n-\t\tfree_op_meta(metabuf, instance->meta_info.pool);\n+\t\tops[i] = otx_cpt_process_response(instance, (void *)ops[i],\n+\t\t\t\t\t\t cc[i], op_type);\n \t}\n \n \treturn nb_completed;\n@@ -925,6 +1013,32 @@ otx_cpt_dequeue_sym(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)\n \treturn otx_cpt_pkt_dequeue(qptr, ops, nb_ops, OP_TYPE_SYM);\n }\n \n+uintptr_t __rte_hot\n+otx_crypto_adapter_dequeue(uintptr_t get_work1)\n+{\n+\tconst struct cpt_instance *instance;\n+\tstruct cpt_request_info *req;\n+\tstruct rte_crypto_op *cop;\n+\tuint8_t cc, op_type;\n+\tuintptr_t *rsp;\n+\n+\treq = (struct cpt_request_info *)get_work1;\n+\tinstance = req->qp;\n+\trsp = req->op;\n+\tcop = (void *)rsp[1];\n+\top_type = cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC ? OP_TYPE_SYM :\n+\t\t\t\t\t\t\t OP_TYPE_ASYM;\n+\n+\tdo {\n+\t\tcc = check_nb_command_id(\n+\t\t\treq, (struct cpt_instance *)(uintptr_t)instance);\n+\t} while (cc == ERR_REQ_PENDING);\n+\n+\tcop = otx_cpt_process_response(instance, (void *)req->op, cc, op_type);\n+\n+\treturn (uintptr_t)(cop);\n+}\n+\n static struct rte_cryptodev_ops cptvf_ops = {\n \t/* Device related operations */\n \t.dev_configure = otx_cpt_dev_config,\ndiff --git a/drivers/crypto/octeontx/otx_cryptodev_ops.h b/drivers/crypto/octeontx/otx_cryptodev_ops.h\nindex fac8a3c006..f234f16970 100644\n--- a/drivers/crypto/octeontx/otx_cryptodev_ops.h\n+++ b/drivers/crypto/octeontx/otx_cryptodev_ops.h\n@@ -14,4 +14,12 @@\n int\n otx_cpt_dev_create(struct rte_cryptodev *c_dev);\n \n+__rte_internal\n+uint16_t __rte_hot\n+otx_crypto_adapter_enqueue(void *port, struct rte_crypto_op *op);\n+\n+__rte_internal\n+uintptr_t __rte_hot\n+otx_crypto_adapter_dequeue(uintptr_t get_work1);\n+\n #endif /* _OTX_CRYPTODEV_OPS_H_ */\ndiff --git a/drivers/crypto/octeontx/version.map b/drivers/crypto/octeontx/version.map\nindex 4a76d1d52d..41f33a4ecf 100644\n--- a/drivers/crypto/octeontx/version.map\n+++ b/drivers/crypto/octeontx/version.map\n@@ -1,3 +1,12 @@\n DPDK_21 {\n \tlocal: *;\n };\n+\n+INTERNAL {\n+\tglobal:\n+\n+\totx_crypto_adapter_enqueue;\n+\totx_crypto_adapter_dequeue;\n+\n+\tlocal: *;\n+};\ndiff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c\nindex 25bf207db6..b93f6ec8c6 100644\n--- a/drivers/event/octeontx/ssovf_evdev.c\n+++ b/drivers/event/octeontx/ssovf_evdev.c\n@@ -734,7 +734,8 @@ ssovf_crypto_adapter_caps_get(const struct rte_eventdev *dev,\n \tRTE_SET_USED(dev);\n \tRTE_SET_USED(cdev);\n \n-\t*caps = 0;\n+\t*caps = RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD |\n+\t\tRTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA;\n \n \treturn 0;\n }\ndiff --git a/drivers/event/octeontx/ssovf_worker.c b/drivers/event/octeontx/ssovf_worker.c\nindex a9149fb934..8b056ddc5a 100644\n--- a/drivers/event/octeontx/ssovf_worker.c\n+++ b/drivers/event/octeontx/ssovf_worker.c\n@@ -322,6 +322,15 @@ sso_event_tx_adapter_enqueue_ ## name(void *port, struct rte_event ev[], \\\n SSO_TX_ADPTR_ENQ_FASTPATH_FUNC\n #undef T\n \n+static uint16_t __rte_hot\n+ssow_crypto_adapter_enqueue(void *port, struct rte_event ev[],\n+\t\t\t uint16_t nb_events)\n+{\n+\tRTE_SET_USED(nb_events);\n+\n+\treturn otx_crypto_adapter_enqueue(port, ev->event_ptr);\n+}\n+\n void\n ssovf_fastpath_fns_set(struct rte_eventdev *dev)\n {\n@@ -332,6 +341,8 @@ ssovf_fastpath_fns_set(struct rte_eventdev *dev)\n \tdev->enqueue_new_burst = ssows_enq_new_burst;\n \tdev->enqueue_forward_burst = ssows_enq_fwd_burst;\n \n+\tdev->ca_enqueue = ssow_crypto_adapter_enqueue;\n+\n \tconst event_tx_adapter_enqueue ssow_txa_enqueue[2][2][2][2] = {\n #define T(name, f3, f2, f1, f0, sz, flags)\t\t\t\t\\\n \t[f3][f2][f1][f0] = sso_event_tx_adapter_enqueue_ ##name,\ndiff --git a/drivers/event/octeontx/ssovf_worker.h b/drivers/event/octeontx/ssovf_worker.h\nindex 4354f007d7..f609b296ed 100644\n--- a/drivers/event/octeontx/ssovf_worker.h\n+++ b/drivers/event/octeontx/ssovf_worker.h\n@@ -4,6 +4,9 @@\n \n #include <arpa/inet.h>\n \n+#ifndef _SSOVF_WORKER_H_\n+#define _SSOVF_WORKER_H_\n+\n #include <rte_common.h>\n #include <rte_branch_prediction.h>\n \n@@ -11,6 +14,7 @@\n \n #include \"ssovf_evdev.h\"\n #include \"octeontx_rxtx.h\"\n+#include \"otx_cryptodev_ops.h\"\n \n /* Alignment */\n #define OCCTX_ALIGN 128\n@@ -174,14 +178,17 @@ ssows_get_work(struct ssows *ws, struct rte_event *ev, const uint16_t flag)\n \tsched_type_queue = sched_type_queue << 38;\n \tev->event = sched_type_queue | (get_work0 & 0xffffffff);\n \n-\tif (get_work1 && ev->event_type == RTE_EVENT_TYPE_ETHDEV) {\n-\t\tev->mbuf = ssovf_octeontx_wqe_to_pkt(get_work1,\n-\t\t\t\t(ev->event >> 20) & 0x7F, flag, ws->lookup_mem);\n+\tif (get_work1) {\n+\t\tif (ev->event_type == RTE_EVENT_TYPE_ETHDEV)\n+\t\t\tget_work1 = (uintptr_t)ssovf_octeontx_wqe_to_pkt(\n+\t\t\t\tget_work1, (ev->event >> 20) & 0x7F, flag,\n+\t\t\t\tws->lookup_mem);\n+\t\telse if (ev->event_type == RTE_EVENT_TYPE_CRYPTODEV)\n+\t\t\tget_work1 = otx_crypto_adapter_dequeue(get_work1);\n+\t\tev->u64 = get_work1;\n \t} else if (unlikely((get_work0 & 0xFFFFFFFF) == 0xFFFFFFFF)) {\n \t\tssovf_octeontx_wqe_free(get_work1);\n \t\treturn 0;\n-\t} else {\n-\t\tev->u64 = get_work1;\n \t}\n \n \treturn !!get_work1;\n@@ -254,3 +261,11 @@ ssows_swtag_wait(struct ssows *ws)\n \twhile (ssovf_read64(ws->base + SSOW_VHWS_SWTP))\n \t;\n }\n+\n+static __rte_always_inline void\n+ssows_head_wait(struct ssows *ws)\n+{\n+\twhile (!(ssovf_read64(ws->base + SSOW_VHWS_TAG) & (1ULL << 35)))\n+\t\t;\n+}\n+#endif /* _SSOVF_WORKER_H_ */\ndiff --git a/drivers/event/octeontx2/otx2_evdev_crypto_adptr_rx.h b/drivers/event/octeontx2/otx2_evdev_crypto_adptr_rx.h\nindex 9e331fdd75..a543225376 100644\n--- a/drivers/event/octeontx2/otx2_evdev_crypto_adptr_rx.h\n+++ b/drivers/event/octeontx2/otx2_evdev_crypto_adptr_rx.h\n@@ -54,6 +54,7 @@ static inline uint64_t\n otx2_handle_crypto_event(uint64_t get_work1)\n {\n \tstruct cpt_request_info *req;\n+\tconst struct otx2_cpt_qp *qp;\n \tstruct rte_crypto_op *cop;\n \tuintptr_t *rsp;\n \tvoid *metabuf;\n@@ -61,14 +62,15 @@ otx2_handle_crypto_event(uint64_t get_work1)\n \n \treq = (struct cpt_request_info *)(get_work1);\n \tcc = otx2_cpt_compcode_get(req);\n+\tqp = req->qp;\n \n \trsp = req->op;\n \tmetabuf = (void *)rsp[0];\n \tcop = (void *)rsp[1];\n \n-\totx2_ca_deq_post_process(req->qp, cop, rsp, cc);\n+\totx2_ca_deq_post_process(qp, cop, rsp, cc);\n \n-\trte_mempool_put(req->qp->meta_info.pool, metabuf);\n+\trte_mempool_put(qp->meta_info.pool, metabuf);\n \n \treturn (uint64_t)(cop);\n }\n", "prefixes": [ "v2", "2/2" ] }