Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/117261/?format=api
{ "id": 117261, "url": "http://patches.dpdk.org/api/patches/117261/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20221003135248.2855396-3-gakhil@marvell.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20221003135248.2855396-3-gakhil@marvell.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20221003135248.2855396-3-gakhil@marvell.com", "date": "2022-10-03T13:52:44", "name": "[v6,2/6] crypto/scheduler: use unified session", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "aa627d8572e88410fd3cc70760a6b92e8f035c7f", "submitter": { "id": 2094, "url": "http://patches.dpdk.org/api/people/2094/?format=api", "name": "Akhil Goyal", "email": "gakhil@marvell.com" }, "delegate": { "id": 6690, "url": "http://patches.dpdk.org/api/users/6690/?format=api", "username": "akhil", "first_name": "akhil", "last_name": "goyal", "email": "gakhil@marvell.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20221003135248.2855396-3-gakhil@marvell.com/mbox/", "series": [ { "id": 24948, "url": "http://patches.dpdk.org/api/series/24948/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=24948", "date": "2022-10-03T13:52:42", "name": "crypto/security session framework rework", "version": 6, "mbox": "http://patches.dpdk.org/series/24948/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/117261/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/117261/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 5689BA00C3;\n\tMon, 3 Oct 2022 15:53:36 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id BA0E0427F3;\n\tMon, 3 Oct 2022 15:53:30 +0200 (CEST)", "from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com\n [67.231.156.173])\n by mails.dpdk.org (Postfix) with ESMTP id 6F70F427F3\n for <dev@dpdk.org>; Mon, 3 Oct 2022 15:53:28 +0200 (CEST)", "from pps.filterd (m0045851.ppops.net [127.0.0.1])\n by mx0b-0016f401.pphosted.com (8.17.1.5/8.17.1.5) with ESMTP id\n 292MiJEj019108;\n Mon, 3 Oct 2022 06:53:23 -0700", "from dc5-exch01.marvell.com ([199.233.59.181])\n by mx0b-0016f401.pphosted.com (PPS) with ESMTPS id 3jxn2pnp16-1\n (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT);\n Mon, 03 Oct 2022 06:53:23 -0700", "from DC5-EXCH01.marvell.com (10.69.176.38) by DC5-EXCH01.marvell.com\n (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.2;\n Mon, 3 Oct 2022 06:53:21 -0700", "from maili.marvell.com (10.69.176.80) by DC5-EXCH01.marvell.com\n (10.69.176.38) with Microsoft SMTP Server id 15.0.1497.2 via Frontend\n Transport; Mon, 3 Oct 2022 06:53:21 -0700", "from localhost.localdomain (unknown [10.28.36.102])\n by maili.marvell.com (Postfix) with ESMTP id AD2D53F704C;\n Mon, 3 Oct 2022 06:53:11 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com;\n h=from : to : cc :\n subject : date : message-id : in-reply-to : references : mime-version :\n content-transfer-encoding : content-type; s=pfpt0220;\n bh=sTk7JBI/zze/JgpZl6OJYCICJqUcZrvWRfIluvIDRMs=;\n b=IGWuSdcO54tlQm/Nvef746zQ/f//Ogo3yhVC6VRtNEXrEunyi6BMaAhCp1y/P5kd/nfM\n Mi9oZQmI1YIIgBRP7g68L+n1WkyPjR2KPlsaLnkObTq9UxGw4mmestRfO4O0Uniy9S04\n uzjSySUkKgWPV3DXOs6GNKqUWAjc3w2s8OJFqhsIWSdGo0/JXmakQcFFSFEu/VYcBH2N\n JgEOWGP33ryxnRg9Qq2QyCuEkZek2+X7Gl2t3lqWSAsinV9paldGg6LSt4L5ZM3ZOgMH\n MXvXqm5/zPPfKRUSfz9cK3pkt+aj4VuehhbmU7fokk23gcSkCY9fbwOy9Dy38t5bb3K4 Lg==", "From": "Akhil Goyal <gakhil@marvell.com>", "To": "<dev@dpdk.org>", "CC": "<thomas@monjalon.net>, <david.marchand@redhat.com>,\n <hemant.agrawal@nxp.com>, <vattunuru@marvell.com>,\n <ferruh.yigit@xilinx.com>, <andrew.rybchenko@oktetlabs.ru>,\n <konstantin.v.ananyev@yandex.ru>, <jiawenwu@trustnetic.com>,\n <yisen.zhuang@huawei.com>, <irusskikh@marvell.com>,\n <jerinj@marvell.com>, <adwivedi@marvell.com>,\n <maxime.coquelin@redhat.com>, <chandu@amd.com>, <ruifeng.wang@arm.com>,\n <ajit.khaparde@broadcom.com>, <anoobj@marvell.com>,\n <pablo.de.lara.guarch@intel.com>, <matan@nvidia.com>,\n <g.singh@nxp.com>, <qiming.yang@intel.com>, <wenjun1.wu@intel.com>,\n <jianwang@trustnetic.com>, <jingjing.wu@intel.com>,\n <beilei.xing@intel.com>, <ndabilpuram@marvell.com>,\n <roy.fan.zhang@intel.com>, <lironh@marvell.com>,\n <royzhang1980@gmail.com>, <sunilprakashrao.uttarwar@amd.com>,\n <kai.ji@intel.com>, <rnagadheeraj@marvell.com>,\n <jianjay.zhou@huawei.com>, Akhil Goyal <gakhil@marvell.com>,\n David Coyle <david.coyle@intel.com>, Kevin O'Sullivan\n <kevin.osullivan@intel.com>", "Subject": "[PATCH v6 2/6] crypto/scheduler: use unified session", "Date": "Mon, 3 Oct 2022 19:22:44 +0530", "Message-ID": "<20221003135248.2855396-3-gakhil@marvell.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20221003135248.2855396-1-gakhil@marvell.com>", "References": "<20221003135248.2855396-1-gakhil@marvell.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Content-Type": "text/plain", "X-Proofpoint-ORIG-GUID": "VP3TB2-pav6GGb5Din29qLb0sIfJt8Qp", "X-Proofpoint-GUID": "VP3TB2-pav6GGb5Din29qLb0sIfJt8Qp", "X-Proofpoint-Virus-Version": "vendor=baseguard\n engine=ICAP:2.0.205,Aquarius:18.0.895,Hydra:6.0.528,FMLib:17.11.122.1\n definitions=2022-10-03_02,2022-09-29_03,2022-06-22_01", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "From: Fan Zhang <roy.fan.zhang@intel.com>\n\nThis patch updates the scheduler PMD to use unified session\ndata structure. Previously thanks to the private session\narray in cryptodev sym session there are no necessary\nchange needed for scheduler PMD other than the way ops\nare enqueued/dequeued. The patch inherits the same design\nin the original session data structure to the scheduler PMD\nso the cryptodev sym session can be as a linear buffer for\nboth session header and driver private data.\n\nWith the change there are inevitable extra cost on both memory\n(64 bytes per session per driver type) and cycle count (set\nthe correct session for each cop based on the worker before\nenqueue, and retrieve the original session after dequeue).\n\nSigned-off-by: Fan Zhang <roy.fan.zhang@intel.com>\nSigned-off-by: Akhil Goyal <gakhil@marvell.com>\nAcked-by: Kai Ji <kai.ji@intel.com>\nTested-by: Gagandeep Singh <g.singh@nxp.com>\nTested-by: David Coyle <david.coyle@intel.com>\nTested-by: Kevin O'Sullivan <kevin.osullivan@intel.com>\n---\n drivers/crypto/scheduler/scheduler_failover.c | 19 ++-\n .../crypto/scheduler/scheduler_multicore.c | 17 +++\n .../scheduler/scheduler_pkt_size_distr.c | 84 +++++++++---\n drivers/crypto/scheduler/scheduler_pmd_ops.c | 107 +++++++++++++++-\n .../crypto/scheduler/scheduler_pmd_private.h | 120 +++++++++++++++++-\n .../crypto/scheduler/scheduler_roundrobin.c | 11 +-\n 6 files changed, 318 insertions(+), 40 deletions(-)", "diff": "diff --git a/drivers/crypto/scheduler/scheduler_failover.c b/drivers/crypto/scheduler/scheduler_failover.c\nindex 2a0e29fa72..7fadcf66d0 100644\n--- a/drivers/crypto/scheduler/scheduler_failover.c\n+++ b/drivers/crypto/scheduler/scheduler_failover.c\n@@ -16,18 +16,19 @@\n struct fo_scheduler_qp_ctx {\n \tstruct scheduler_worker primary_worker;\n \tstruct scheduler_worker secondary_worker;\n+\tuint8_t primary_worker_index;\n+\tuint8_t secondary_worker_index;\n \n \tuint8_t deq_idx;\n };\n \n static __rte_always_inline uint16_t\n failover_worker_enqueue(struct scheduler_worker *worker,\n-\t\tstruct rte_crypto_op **ops, uint16_t nb_ops)\n+\t\tstruct rte_crypto_op **ops, uint16_t nb_ops, uint8_t index)\n {\n-\tuint16_t i, processed_ops;\n+\tuint16_t processed_ops;\n \n-\tfor (i = 0; i < nb_ops && i < 4; i++)\n-\t\trte_prefetch0(ops[i]->sym->session);\n+\tscheduler_set_worker_session(ops, nb_ops, index);\n \n \tprocessed_ops = rte_cryptodev_enqueue_burst(worker->dev_id,\n \t\t\tworker->qp_id, ops, nb_ops);\n@@ -47,13 +48,14 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t\treturn 0;\n \n \tenqueued_ops = failover_worker_enqueue(&qp_ctx->primary_worker,\n-\t\t\tops, nb_ops);\n+\t\t\tops, nb_ops, PRIMARY_WORKER_IDX);\n \n \tif (enqueued_ops < nb_ops)\n \t\tenqueued_ops += failover_worker_enqueue(\n \t\t\t\t&qp_ctx->secondary_worker,\n \t\t\t\t&ops[enqueued_ops],\n-\t\t\t\tnb_ops - enqueued_ops);\n+\t\t\t\tnb_ops - enqueued_ops,\n+\t\t\t\tSECONDARY_WORKER_IDX);\n \n \treturn enqueued_ops;\n }\n@@ -94,7 +96,7 @@ schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \tqp_ctx->deq_idx = (~qp_ctx->deq_idx) & WORKER_SWITCH_MASK;\n \n \tif (nb_deq_ops == nb_ops)\n-\t\treturn nb_deq_ops;\n+\t\tgoto retrieve_session;\n \n \tworker = workers[qp_ctx->deq_idx];\n \n@@ -104,6 +106,9 @@ schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t\tworker->nb_inflight_cops -= nb_deq_ops2;\n \t}\n \n+retrieve_session:\n+\tscheduler_retrieve_session(ops, nb_deq_ops + nb_deq_ops2);\n+\n \treturn nb_deq_ops + nb_deq_ops2;\n }\n \ndiff --git a/drivers/crypto/scheduler/scheduler_multicore.c b/drivers/crypto/scheduler/scheduler_multicore.c\nindex 900ab4049d..3dea850661 100644\n--- a/drivers/crypto/scheduler/scheduler_multicore.c\n+++ b/drivers/crypto/scheduler/scheduler_multicore.c\n@@ -183,11 +183,19 @@ mc_scheduler_worker(struct rte_cryptodev *dev)\n \n \twhile (!mc_ctx->stop_signal) {\n \t\tif (pending_enq_ops) {\n+\t\t\tscheduler_set_worker_session(\n+\t\t\t\t&enq_ops[pending_enq_ops_idx], pending_enq_ops,\n+\t\t\t\tworker_idx);\n \t\t\tprocessed_ops =\n \t\t\t\trte_cryptodev_enqueue_burst(worker->dev_id,\n \t\t\t\t\tworker->qp_id,\n \t\t\t\t\t&enq_ops[pending_enq_ops_idx],\n \t\t\t\t\tpending_enq_ops);\n+\t\t\tif (processed_ops < pending_deq_ops)\n+\t\t\t\tscheduler_retrieve_session(\n+\t\t\t\t\t&enq_ops[pending_enq_ops_idx +\n+\t\t\t\t\t\tprocessed_ops],\n+\t\t\t\t\tpending_deq_ops - processed_ops);\n \t\t\tpending_enq_ops -= processed_ops;\n \t\t\tpending_enq_ops_idx += processed_ops;\n \t\t\tinflight_ops += processed_ops;\n@@ -195,9 +203,16 @@ mc_scheduler_worker(struct rte_cryptodev *dev)\n \t\t\tprocessed_ops = rte_ring_dequeue_burst(enq_ring, (void *)enq_ops,\n \t\t\t\t\t\t\tMC_SCHED_BUFFER_SIZE, NULL);\n \t\t\tif (processed_ops) {\n+\t\t\t\tscheduler_set_worker_session(enq_ops,\n+\t\t\t\t\tprocessed_ops, worker_idx);\n \t\t\t\tpending_enq_ops_idx = rte_cryptodev_enqueue_burst(\n \t\t\t\t\t\tworker->dev_id, worker->qp_id,\n \t\t\t\t\t\tenq_ops, processed_ops);\n+\t\t\t\tif (pending_enq_ops_idx < processed_ops)\n+\t\t\t\t\tscheduler_retrieve_session(\n+\t\t\t\t\t\tenq_ops + pending_enq_ops_idx,\n+\t\t\t\t\t\tprocessed_ops -\n+\t\t\t\t\t\tpending_enq_ops_idx);\n \t\t\t\tpending_enq_ops = processed_ops - pending_enq_ops_idx;\n \t\t\t\tinflight_ops += pending_enq_ops_idx;\n \t\t\t}\n@@ -214,6 +229,8 @@ mc_scheduler_worker(struct rte_cryptodev *dev)\n \t\t\t\t\tworker->dev_id, worker->qp_id, deq_ops,\n \t\t\t\t\tMC_SCHED_BUFFER_SIZE);\n \t\t\tif (processed_ops) {\n+\t\t\t\tscheduler_retrieve_session(deq_ops,\n+\t\t\t\t\tprocessed_ops);\n \t\t\t\tinflight_ops -= processed_ops;\n \t\t\t\tif (reordering_enabled) {\n \t\t\t\t\tuint16_t j;\ndiff --git a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c\nindex 933a5c6978..9204f6f608 100644\n--- a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c\n+++ b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c\n@@ -48,34 +48,54 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t};\n \tstruct psd_schedule_op *p_enq_op;\n \tuint16_t i, processed_ops_pri = 0, processed_ops_sec = 0;\n-\tuint32_t job_len;\n \n \tif (unlikely(nb_ops == 0))\n \t\treturn 0;\n \n \tfor (i = 0; i < nb_ops && i < 4; i++) {\n \t\trte_prefetch0(ops[i]->sym);\n-\t\trte_prefetch0(ops[i]->sym->session);\n+\t\trte_prefetch0((uint8_t *)ops[i]->sym->session +\n+\t\t\tsizeof(struct rte_cryptodev_sym_session));\n \t}\n \n \tfor (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) {\n+\t\tstruct scheduler_session_ctx *sess_ctx[4];\n+\t\tuint8_t target[4];\n+\t\tuint32_t job_len[4];\n+\n \t\trte_prefetch0(ops[i + 4]->sym);\n-\t\trte_prefetch0(ops[i + 4]->sym->session);\n+\t\trte_prefetch0((uint8_t *)ops[i + 4]->sym->session +\n+\t\t\tsizeof(struct rte_cryptodev_sym_session));\n \t\trte_prefetch0(ops[i + 5]->sym);\n-\t\trte_prefetch0(ops[i + 5]->sym->session);\n+\t\trte_prefetch0((uint8_t *)ops[i + 5]->sym->session +\n+\t\t\tsizeof(struct rte_cryptodev_sym_session));\n \t\trte_prefetch0(ops[i + 6]->sym);\n-\t\trte_prefetch0(ops[i + 6]->sym->session);\n+\t\trte_prefetch0((uint8_t *)ops[i + 6]->sym->session +\n+\t\t\tsizeof(struct rte_cryptodev_sym_session));\n \t\trte_prefetch0(ops[i + 7]->sym);\n-\t\trte_prefetch0(ops[i + 7]->sym->session);\n+\t\trte_prefetch0((uint8_t *)ops[i + 7]->sym->session +\n+\t\t\tsizeof(struct rte_cryptodev_sym_session));\n+\n+\t\tsess_ctx[0] = (void *)ops[i]->sym->session->driver_priv_data;\n+\t\tsess_ctx[1] =\n+\t\t\t(void *)ops[i + 1]->sym->session->driver_priv_data;\n+\t\tsess_ctx[2] =\n+\t\t\t(void *)ops[i + 2]->sym->session->driver_priv_data;\n+\t\tsess_ctx[3] =\n+\t\t\t(void *)ops[i + 3]->sym->session->driver_priv_data;\n \n \t\t/* job_len is initialized as cipher data length, once\n \t\t * it is 0, equals to auth data length\n \t\t */\n-\t\tjob_len = ops[i]->sym->cipher.data.length;\n-\t\tjob_len += (ops[i]->sym->cipher.data.length == 0) *\n+\t\tjob_len[0] = ops[i]->sym->cipher.data.length;\n+\t\tjob_len[0] += (ops[i]->sym->cipher.data.length == 0) *\n \t\t\t\tops[i]->sym->auth.data.length;\n \t\t/* decide the target op based on the job length */\n-\t\tp_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];\n+\t\ttarget[0] = !(job_len[0] & psd_qp_ctx->threshold);\n+\t\tif (ops[i]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\tops[i]->sym->session =\n+\t\t\t\tsess_ctx[0]->worker_sess[target[0]];\n+\t\tp_enq_op = &enq_ops[target[0]];\n \n \t\t/* stop schedule cops before the queue is full, this shall\n \t\t * prevent the failed enqueue\n@@ -89,10 +109,14 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t\tsched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i];\n \t\tp_enq_op->pos++;\n \n-\t\tjob_len = ops[i+1]->sym->cipher.data.length;\n-\t\tjob_len += (ops[i+1]->sym->cipher.data.length == 0) *\n+\t\tjob_len[1] = ops[i + 1]->sym->cipher.data.length;\n+\t\tjob_len[1] += (ops[i + 1]->sym->cipher.data.length == 0) *\n \t\t\t\tops[i+1]->sym->auth.data.length;\n-\t\tp_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];\n+\t\ttarget[1] = !(job_len[1] & psd_qp_ctx->threshold);\n+\t\tif (ops[i + 1]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\tops[i + 1]->sym->session =\n+\t\t\t\tsess_ctx[1]->worker_sess[target[1]];\n+\t\tp_enq_op = &enq_ops[target[1]];\n \n \t\tif (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==\n \t\t\t\tqp_ctx->max_nb_objs) {\n@@ -103,10 +127,14 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t\tsched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i+1];\n \t\tp_enq_op->pos++;\n \n-\t\tjob_len = ops[i+2]->sym->cipher.data.length;\n-\t\tjob_len += (ops[i+2]->sym->cipher.data.length == 0) *\n-\t\t\t\tops[i+2]->sym->auth.data.length;\n-\t\tp_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];\n+\t\tjob_len[2] = ops[i + 2]->sym->cipher.data.length;\n+\t\tjob_len[2] += (ops[i + 2]->sym->cipher.data.length == 0) *\n+\t\t\t\tops[i + 2]->sym->auth.data.length;\n+\t\ttarget[2] = !(job_len[2] & psd_qp_ctx->threshold);\n+\t\tif (ops[i + 2]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\tops[i + 2]->sym->session =\n+\t\t\t\tsess_ctx[2]->worker_sess[target[2]];\n+\t\tp_enq_op = &enq_ops[target[2]];\n \n \t\tif (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==\n \t\t\t\tqp_ctx->max_nb_objs) {\n@@ -117,10 +145,14 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t\tsched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i+2];\n \t\tp_enq_op->pos++;\n \n-\t\tjob_len = ops[i+3]->sym->cipher.data.length;\n-\t\tjob_len += (ops[i+3]->sym->cipher.data.length == 0) *\n-\t\t\t\tops[i+3]->sym->auth.data.length;\n-\t\tp_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];\n+\t\tjob_len[3] = ops[i + 3]->sym->cipher.data.length;\n+\t\tjob_len[3] += (ops[i + 3]->sym->cipher.data.length == 0) *\n+\t\t\t\tops[i + 3]->sym->auth.data.length;\n+\t\ttarget[3] = !(job_len[3] & psd_qp_ctx->threshold);\n+\t\tif (ops[i + 3]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\tops[i + 3]->sym->session =\n+\t\t\t\tsess_ctx[1]->worker_sess[target[3]];\n+\t\tp_enq_op = &enq_ops[target[3]];\n \n \t\tif (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==\n \t\t\t\tqp_ctx->max_nb_objs) {\n@@ -133,10 +165,18 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t}\n \n \tfor (; i < nb_ops; i++) {\n+\t\tstruct scheduler_session_ctx *sess_ctx =\n+\t\t\t(void *)ops[i]->sym->session->driver_priv_data;\n+\t\tuint32_t job_len;\n+\t\tuint8_t target;\n+\n \t\tjob_len = ops[i]->sym->cipher.data.length;\n \t\tjob_len += (ops[i]->sym->cipher.data.length == 0) *\n \t\t\t\tops[i]->sym->auth.data.length;\n-\t\tp_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];\n+\t\ttarget = !(job_len & psd_qp_ctx->threshold);\n+\t\tif (ops[i]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\tops[i]->sym->session = sess_ctx->worker_sess[target];\n+\t\tp_enq_op = &enq_ops[target];\n \n \t\tif (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==\n \t\t\t\tqp_ctx->max_nb_objs) {\n@@ -199,6 +239,7 @@ schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \tif (worker->nb_inflight_cops) {\n \t\tnb_deq_ops_pri = rte_cryptodev_dequeue_burst(worker->dev_id,\n \t\t\tworker->qp_id, ops, nb_ops);\n+\t\tscheduler_retrieve_session(ops, nb_deq_ops_pri);\n \t\tworker->nb_inflight_cops -= nb_deq_ops_pri;\n \t}\n \n@@ -213,6 +254,7 @@ schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t\tnb_deq_ops_sec = rte_cryptodev_dequeue_burst(worker->dev_id,\n \t\t\t\tworker->qp_id, &ops[nb_deq_ops_pri],\n \t\t\t\tnb_ops - nb_deq_ops_pri);\n+\t\tscheduler_retrieve_session(ops, nb_deq_ops_sec);\n \t\tworker->nb_inflight_cops -= nb_deq_ops_sec;\n \n \t\tif (!worker->nb_inflight_cops)\ndiff --git a/drivers/crypto/scheduler/scheduler_pmd_ops.c b/drivers/crypto/scheduler/scheduler_pmd_ops.c\nindex 03df424140..c0da39f934 100644\n--- a/drivers/crypto/scheduler/scheduler_pmd_ops.c\n+++ b/drivers/crypto/scheduler/scheduler_pmd_ops.c\n@@ -9,6 +9,7 @@\n #include <rte_cryptodev.h>\n #include <cryptodev_pmd.h>\n #include <rte_reorder.h>\n+#include <rte_errno.h>\n \n #include \"scheduler_pmd_private.h\"\n \n@@ -469,19 +470,113 @@ scheduler_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)\n \treturn max_priv_sess_size;\n }\n \n+struct scheduler_configured_sess_info {\n+\tuint8_t dev_id;\n+\tuint8_t driver_id;\n+\tstruct rte_cryptodev_sym_session *sess;\n+};\n+\n static int\n-scheduler_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused,\n-\tstruct rte_crypto_sym_xform *xform __rte_unused,\n-\tstruct rte_cryptodev_sym_session *sess __rte_unused)\n+scheduler_pmd_sym_session_configure(struct rte_cryptodev *dev,\n+\tstruct rte_crypto_sym_xform *xform,\n+\tstruct rte_cryptodev_sym_session *sess)\n {\n+\tstruct scheduler_ctx *sched_ctx = dev->data->dev_private;\n+\tstruct rte_mempool *mp = rte_mempool_from_obj(sess);\n+\tstruct scheduler_session_ctx *sess_ctx = (void *)sess->driver_priv_data;\n+\tstruct scheduler_configured_sess_info configured_sess[\n+\t\t\tRTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS] = { 0 };\n+\tuint32_t i, j, n_configured_sess = 0;\n+\tint ret = 0;\n+\n+\tif (mp == NULL)\n+\t\treturn -EINVAL;\n+\n+\tfor (i = 0; i < sched_ctx->nb_workers; i++) {\n+\t\tstruct scheduler_worker *worker = &sched_ctx->workers[i];\n+\t\tstruct rte_cryptodev_sym_session *worker_sess;\n+\t\tuint8_t next_worker = 0;\n+\n+\t\tfor (j = 0; j < n_configured_sess; j++) {\n+\t\t\tif (configured_sess[j].driver_id ==\n+\t\t\t\t\tworker->driver_id) {\n+\t\t\t\tsess_ctx->worker_sess[i] =\n+\t\t\t\t\tconfigured_sess[j].sess;\n+\t\t\t\tnext_worker = 1;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t}\n+\t\tif (next_worker)\n+\t\t\tcontinue;\n+\n+\t\tif (rte_mempool_avail_count(mp) == 0) {\n+\t\t\tret = -ENOMEM;\n+\t\t\tgoto error_exit;\n+\t\t}\n+\n+\t\tworker_sess = rte_cryptodev_sym_session_create(worker->dev_id,\n+\t\t\txform, mp);\n+\t\tif (worker_sess == NULL) {\n+\t\t\tret = -rte_errno;\n+\t\t\tgoto error_exit;\n+\t\t}\n+\n+\t\tworker_sess->opaque_data = (uint64_t)sess;\n+\t\tsess_ctx->worker_sess[i] = worker_sess;\n+\t\tconfigured_sess[n_configured_sess].driver_id =\n+\t\t\tworker->driver_id;\n+\t\tconfigured_sess[n_configured_sess].dev_id = worker->dev_id;\n+\t\tconfigured_sess[n_configured_sess].sess = worker_sess;\n+\t\tn_configured_sess++;\n+\t}\n+\n \treturn 0;\n+error_exit:\n+\tsess_ctx->ref_cnt = sched_ctx->ref_cnt;\n+\tfor (i = 0; i < n_configured_sess; i++)\n+\t\trte_cryptodev_sym_session_free(configured_sess[i].dev_id,\n+\t\t\tconfigured_sess[i].sess);\n+\treturn ret;\n }\n \n /** Clear the memory of session so it doesn't leave key material behind */\n static void\n-scheduler_pmd_sym_session_clear(struct rte_cryptodev *dev __rte_unused,\n-\t\tstruct rte_cryptodev_sym_session *sess __rte_unused)\n-{}\n+scheduler_pmd_sym_session_clear(struct rte_cryptodev *dev,\n+\t\tstruct rte_cryptodev_sym_session *sess)\n+{\n+\tstruct scheduler_ctx *sched_ctx = dev->data->dev_private;\n+\tstruct scheduler_session_ctx *sess_ctx = (void *)sess->driver_priv_data;\n+\tstruct scheduler_configured_sess_info deleted_sess[\n+\t\t\tRTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS] = { 0 };\n+\tuint32_t i, j, n_deleted_sess = 0;\n+\n+\tif (sched_ctx->ref_cnt != sess_ctx->ref_cnt) {\n+\t\tCR_SCHED_LOG(WARNING,\n+\t\t\t\"Worker updated between session creation/deletion. \"\n+\t\t\t\"The session may not be freed fully.\");\n+\t}\n+\n+\tfor (i = 0; i < sched_ctx->nb_workers; i++) {\n+\t\tstruct scheduler_worker *worker = &sched_ctx->workers[i];\n+\t\tuint8_t next_worker = 0;\n+\n+\t\tfor (j = 0; j < n_deleted_sess; j++) {\n+\t\t\tif (deleted_sess[j].driver_id == worker->driver_id) {\n+\t\t\t\tsess_ctx->worker_sess[i] = NULL;\n+\t\t\t\tnext_worker = 1;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t}\n+\t\tif (next_worker)\n+\t\t\tcontinue;\n+\n+\t\trte_cryptodev_sym_session_free(worker->dev_id,\n+\t\t\tsess_ctx->worker_sess[i]);\n+\n+\t\tdeleted_sess[n_deleted_sess++].driver_id = worker->driver_id;\n+\t\tsess_ctx->worker_sess[i] = NULL;\n+\t}\n+}\n \n static struct rte_cryptodev_ops scheduler_pmd_ops = {\n \t\t.dev_configure\t\t= scheduler_pmd_config,\ndiff --git a/drivers/crypto/scheduler/scheduler_pmd_private.h b/drivers/crypto/scheduler/scheduler_pmd_private.h\nindex 4d33b9ab44..0e508727a4 100644\n--- a/drivers/crypto/scheduler/scheduler_pmd_private.h\n+++ b/drivers/crypto/scheduler/scheduler_pmd_private.h\n@@ -22,7 +22,6 @@ struct scheduler_worker {\n \tuint8_t dev_id;\n \tuint16_t qp_id;\n \tuint32_t nb_inflight_cops;\n-\n \tuint8_t driver_id;\n };\n \n@@ -37,6 +36,8 @@ struct scheduler_ctx {\n \n \tstruct scheduler_worker workers[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS];\n \tuint32_t nb_workers;\n+\t/* reference count when the workers are incremented/decremented */\n+\tuint32_t ref_cnt;\n \n \tenum rte_cryptodev_scheduler_mode mode;\n \n@@ -61,6 +62,11 @@ struct scheduler_qp_ctx {\n \tstruct rte_ring *order_ring;\n } __rte_cache_aligned;\n \n+struct scheduler_session_ctx {\n+\tuint32_t ref_cnt;\n+\tstruct rte_cryptodev_sym_session *worker_sess[\n+\t\tRTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS];\n+};\n \n extern uint8_t cryptodev_scheduler_driver_id;\n \n@@ -101,6 +107,118 @@ scheduler_order_drain(struct rte_ring *order_ring,\n \treturn nb_ops_to_deq;\n }\n \n+static __rte_always_inline void\n+scheduler_set_worker_session(struct rte_crypto_op **ops, uint16_t nb_ops,\n+\t\tuint8_t worker_index)\n+{\n+\tstruct rte_crypto_op **op = ops;\n+\tuint16_t n = nb_ops;\n+\n+\tif (n >= 4) {\n+\t\trte_prefetch0(op[0]->sym->session);\n+\t\trte_prefetch0(op[1]->sym->session);\n+\t\trte_prefetch0(op[2]->sym->session);\n+\t\trte_prefetch0(op[3]->sym->session);\n+\t}\n+\n+\twhile (n >= 4) {\n+\t\tif (n >= 8) {\n+\t\t\trte_prefetch0(op[4]->sym->session);\n+\t\t\trte_prefetch0(op[5]->sym->session);\n+\t\t\trte_prefetch0(op[6]->sym->session);\n+\t\t\trte_prefetch0(op[7]->sym->session);\n+\t\t}\n+\n+\t\tif (op[0]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {\n+\t\t\tstruct scheduler_session_ctx *sess_ctx =\n+\t\t\t\t(void *)op[0]->sym->session->driver_priv_data;\n+\t\t\top[0]->sym->session =\n+\t\t\t\tsess_ctx->worker_sess[worker_index];\n+\t\t}\n+\n+\t\tif (op[1]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {\n+\t\t\tstruct scheduler_session_ctx *sess_ctx =\n+\t\t\t\t(void *)op[1]->sym->session->driver_priv_data;\n+\t\t\top[1]->sym->session =\n+\t\t\t\tsess_ctx->worker_sess[worker_index];\n+\t\t}\n+\n+\t\tif (op[2]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {\n+\t\t\tstruct scheduler_session_ctx *sess_ctx =\n+\t\t\t\t(void *)op[2]->sym->session->driver_priv_data;\n+\t\t\top[2]->sym->session =\n+\t\t\t\tsess_ctx->worker_sess[worker_index];\n+\t\t}\n+\n+\t\tif (op[3]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {\n+\t\t\tstruct scheduler_session_ctx *sess_ctx =\n+\t\t\t\t(void *)op[3]->sym->session->driver_priv_data;\n+\t\t\top[3]->sym->session =\n+\t\t\t\tsess_ctx->worker_sess[worker_index];\n+\t\t}\n+\n+\t\top += 4;\n+\t\tn -= 4;\n+\t}\n+\n+\twhile (n--) {\n+\t\tif (op[0]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {\n+\t\t\tstruct scheduler_session_ctx *sess_ctx =\n+\t\t\t\t(void *)op[0]->sym->session->driver_priv_data;\n+\n+\t\t\top[0]->sym->session =\n+\t\t\t\tsess_ctx->worker_sess[worker_index];\n+\t\t\top++;\n+\t\t}\n+\t}\n+}\n+\n+static __rte_always_inline void\n+scheduler_retrieve_session(struct rte_crypto_op **ops, uint16_t nb_ops)\n+{\n+\tuint16_t n = nb_ops;\n+\tstruct rte_crypto_op **op = ops;\n+\n+\tif (n >= 4) {\n+\t\trte_prefetch0(op[0]->sym->session);\n+\t\trte_prefetch0(op[1]->sym->session);\n+\t\trte_prefetch0(op[2]->sym->session);\n+\t\trte_prefetch0(op[3]->sym->session);\n+\t}\n+\n+\twhile (n >= 4) {\n+\t\tif (n >= 8) {\n+\t\t\trte_prefetch0(op[4]->sym->session);\n+\t\t\trte_prefetch0(op[5]->sym->session);\n+\t\t\trte_prefetch0(op[6]->sym->session);\n+\t\t\trte_prefetch0(op[7]->sym->session);\n+\t\t}\n+\n+\t\tif (op[0]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\top[0]->sym->session =\n+\t\t\t\t(void *)op[0]->sym->session->opaque_data;\n+\t\tif (op[1]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\top[1]->sym->session =\n+\t\t\t\t(void *)op[1]->sym->session->opaque_data;\n+\t\tif (op[2]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\top[2]->sym->session =\n+\t\t\t\t(void *)op[2]->sym->session->opaque_data;\n+\t\tif (op[3]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\top[3]->sym->session =\n+\t\t\t\t(void *)op[3]->sym->session->opaque_data;\n+\n+\t\top += 4;\n+\t\tn -= 4;\n+\t}\n+\n+\twhile (n--) {\n+\t\tif (op[0]->sess_type == RTE_CRYPTO_OP_WITH_SESSION)\n+\t\t\top[0]->sym->session =\n+\t\t\t\t(void *)op[0]->sym->session->opaque_data;\n+\t\top++;\n+\t}\n+}\n+\n /** device specific operations function pointer structure */\n extern struct rte_cryptodev_ops *rte_crypto_scheduler_pmd_ops;\n \ndiff --git a/drivers/crypto/scheduler/scheduler_roundrobin.c b/drivers/crypto/scheduler/scheduler_roundrobin.c\nindex ace2dec2ec..ad3f8b842a 100644\n--- a/drivers/crypto/scheduler/scheduler_roundrobin.c\n+++ b/drivers/crypto/scheduler/scheduler_roundrobin.c\n@@ -23,16 +23,17 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \t\t\t((struct scheduler_qp_ctx *)qp)->private_qp_ctx;\n \tuint32_t worker_idx = rr_qp_ctx->last_enq_worker_idx;\n \tstruct scheduler_worker *worker = &rr_qp_ctx->workers[worker_idx];\n-\tuint16_t i, processed_ops;\n+\tuint16_t processed_ops;\n \n \tif (unlikely(nb_ops == 0))\n \t\treturn 0;\n \n-\tfor (i = 0; i < nb_ops && i < 4; i++)\n-\t\trte_prefetch0(ops[i]->sym->session);\n-\n+\tscheduler_set_worker_session(ops, nb_ops, worker_idx);\n \tprocessed_ops = rte_cryptodev_enqueue_burst(worker->dev_id,\n \t\t\tworker->qp_id, ops, nb_ops);\n+\tif (processed_ops < nb_ops)\n+\t\tscheduler_retrieve_session(ops + processed_ops,\n+\t\t\tnb_ops - processed_ops);\n \n \tworker->nb_inflight_cops += processed_ops;\n \n@@ -86,7 +87,7 @@ schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n \n \tnb_deq_ops = rte_cryptodev_dequeue_burst(worker->dev_id,\n \t\t\tworker->qp_id, ops, nb_ops);\n-\n+\tscheduler_retrieve_session(ops, nb_deq_ops);\n \tlast_worker_idx += 1;\n \tlast_worker_idx %= rr_qp_ctx->nb_workers;\n \n", "prefixes": [ "v6", "2/6" ] }