Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/95899/?format=api
{ "id": 95899, "url": "http://patches.dpdk.org/api/patches/95899/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20210715150817.51485-13-shirik@nvidia.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20210715150817.51485-13-shirik@nvidia.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20210715150817.51485-13-shirik@nvidia.com", "date": "2021-07-15T15:08:13", "name": "[v7,12/16] crypto/mlx5: add enqueue and dequeue operations", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "5ccff423f00dc9758f017ac547f2623243e4c5bd", "submitter": { "id": 1894, "url": "http://patches.dpdk.org/api/people/1894/?format=api", "name": "Shiri Kuzin", "email": "shirik@nvidia.com" }, "delegate": { "id": 6690, "url": "http://patches.dpdk.org/api/users/6690/?format=api", "username": "akhil", "first_name": "akhil", "last_name": "goyal", "email": "gakhil@marvell.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20210715150817.51485-13-shirik@nvidia.com/mbox/", "series": [ { "id": 17840, "url": "http://patches.dpdk.org/api/series/17840/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=17840", "date": "2021-07-15T15:08:01", "name": "drivers: introduce mlx5 crypto PMD", "version": 7, "mbox": "http://patches.dpdk.org/series/17840/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/95899/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/95899/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 8EEFCA0A0C;\n\tThu, 15 Jul 2021 17:10:45 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 66F4F41278;\n\tThu, 15 Jul 2021 17:09:35 +0200 (CEST)", "from NAM11-DM6-obe.outbound.protection.outlook.com\n (mail-dm6nam11on2055.outbound.protection.outlook.com [40.107.223.55])\n by mails.dpdk.org (Postfix) with ESMTP id D485141300\n for <dev@dpdk.org>; Thu, 15 Jul 2021 17:09:25 +0200 (CEST)", "from MW4PR04CA0079.namprd04.prod.outlook.com (2603:10b6:303:6b::24)\n by BN6PR12MB1233.namprd12.prod.outlook.com (2603:10b6:404:1c::19)\n with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.4308.23; Thu, 15 Jul\n 2021 15:09:23 +0000", "from CO1NAM11FT015.eop-nam11.prod.protection.outlook.com\n (2603:10b6:303:6b:cafe::9d) by MW4PR04CA0079.outlook.office365.com\n (2603:10b6:303:6b::24) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.4331.23 via Frontend\n Transport; Thu, 15 Jul 2021 15:09:23 +0000", "from mail.nvidia.com (216.228.112.34) by\n CO1NAM11FT015.mail.protection.outlook.com (10.13.175.130) with Microsoft SMTP\n Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id\n 15.20.4331.21 via Frontend Transport; Thu, 15 Jul 2021 15:09:22 +0000", "from nvidia.com (172.20.187.6) by HQMAIL107.nvidia.com\n (172.20.187.13) with Microsoft SMTP Server (TLS) id 15.0.1497.2; Thu, 15 Jul\n 2021 15:09:20 +0000" ], "ARC-Seal": "i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none;\n b=IsXu1rsEfy+B+W1OPGKGXQo7oM4Enl3q2MZpkqFb6ddj+J+3ACwqbNZStF3q+BiNtiOB7q7jVTKCpqdalExpumPlDcdFecgIBk23jbqQRTWe7n6IDyi0zCEubZ6i+fYXtrbH8Y1yS8wEBlW9dxel+SqYc/4LvoxjkkBP6sJmiimYeCRJmtBMYf/jxit/YlxlJ1QrnPqT4ghENOl/ov2meR4EvIDkXbjAEWkrRsL/VelLIQvUBdG6A8/+7yoVIofeqxHU/tCd7RJfLt4Qf8LaVurYhSxKe0FTuiBBC/14S3sMFuzD4uYwoDZOq9lyx3tCyjxwwlmfg9AJ5PYnyM6wEA==", "ARC-Message-Signature": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com;\n s=arcselector9901;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=S6dGTk2rSEm4EVTpfADNvh0e8eNklBib34M21ZVJkCk=;\n b=f4lOLaDP4Q4xN6/cGbUFFOuDzj4d/qDXkCj/8o70UmZg8FHCqFHvoO9YUBonN62QUL4/j0yX+VgPTjS8w5ly9Ph/McIBtVG7xpI8U2IDV8qim6Rj3u9Fm2zH+5eFR7skiGNnyQCwyJUj+MNK/YqHB9ztlAhBTVyEfTKk5MzkbtikH05VhuFeEgBtuoWasRqFoyAT+ia3D6XF3L+p8MpAamlZs9q72Qp0m7BC9fEK2iOzlZkeYuk1Oigd4jyEhojxzCdlWAgXsLmlxTU31x0qb83W5liyuKSwkRIPVHTh7V8LZBjzZYibLGktVzA4sUGuilTos2JMBg6EJRchOiiEJw==", "ARC-Authentication-Results": "i=1; mx.microsoft.com 1; spf=pass (sender ip is\n 216.228.112.34) smtp.rcpttodomain=redhat.com smtp.mailfrom=nvidia.com;\n dmarc=pass (p=quarantine sp=none pct=100) action=none header.from=nvidia.com;\n dkim=none (message not signed); arc=none", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=Nvidia.com;\n s=selector2;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=S6dGTk2rSEm4EVTpfADNvh0e8eNklBib34M21ZVJkCk=;\n b=UE3j/11nBwgEhJ6VKR2DuRHALI54GuAHQ9T55SP32L4hIXAu4U4cCH1SnnfxuPOV010UKOmwWce+KjFL11otwwC3kB02G9G7EkzLAikdsFX6PcbyvfD9GyurgDw5HOzhR+6T32XjF7Xz/01kiRWjMoLZyGuJ2FSJAcqZBZ2uGu3hky85PU/4ku6u5DdDEzto2K9MAoJxu8qxkszqQSN5OMa5ig0gkgTmsyqp3qb8pEdSNlv0Q9ewjpn7R9nKVrR8DS7wEa0TdZNGW3tjOItBZ69Fw1Fx8vhFTKo4LxvxTAV0pdKcwj4tea6ntLuc97CguWhz8+YnlwQfw1y665cwiA==", "X-MS-Exchange-Authentication-Results": "spf=pass (sender IP is 216.228.112.34)\n smtp.mailfrom=nvidia.com; redhat.com; dkim=none (message not signed)\n header.d=none;redhat.com; dmarc=pass action=none header.from=nvidia.com;", "Received-SPF": "Pass (protection.outlook.com: domain of nvidia.com designates\n 216.228.112.34 as permitted sender) receiver=protection.outlook.com;\n client-ip=216.228.112.34; helo=mail.nvidia.com;", "From": "Shiri Kuzin <shirik@nvidia.com>", "To": "<dev@dpdk.org>", "CC": "<matan@nvidia.com>, <gakhil@marvell.com>, <suanmingm@nvidia.com>,\n <david.marchand@redhat.com>, Michael Baum <michaelba@nvidia.com>", "Date": "Thu, 15 Jul 2021 18:08:13 +0300", "Message-ID": "<20210715150817.51485-13-shirik@nvidia.com>", "X-Mailer": "git-send-email 2.27.0", "In-Reply-To": "<20210715150817.51485-1-shirik@nvidia.com>", "References": "<20210708152530.25835-1-shirik@nvidia.com>\n <20210715150817.51485-1-shirik@nvidia.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Content-Type": "text/plain", "X-Originating-IP": "[172.20.187.6]", "X-ClientProxiedBy": "HQMAIL111.nvidia.com (172.20.187.18) To\n HQMAIL107.nvidia.com (172.20.187.13)", "X-EOPAttributedMessage": "0", "X-MS-PublicTrafficType": "Email", "X-MS-Office365-Filtering-Correlation-Id": "08a6430e-21f3-4284-7972-08d947a28748", "X-MS-TrafficTypeDiagnostic": "BN6PR12MB1233:", "X-Microsoft-Antispam-PRVS": "\n <BN6PR12MB1233F8A871CCA9F99AE80FA8AA129@BN6PR12MB1233.namprd12.prod.outlook.com>", "X-MS-Oob-TLC-OOBClassifiers": "OLM:316;", "X-MS-Exchange-SenderADCheck": "1", "X-MS-Exchange-AntiSpam-Relay": "0", "X-Microsoft-Antispam": "BCL:0;", "X-Microsoft-Antispam-Message-Info": "\n JPWnTjg4/ubX1V3+mk0BdbHlnTZliaNm9b0g0JPfeuHizAYo/Rb6JkaUJqL0IAKPVkDJ+5m4HXsK7a7KQSqnUjXKJa/IT8PfvX/9zXItvfIjSrJa21k4Gc6WvX6NhbBvh/QvYH5Kvt/5PcWKcxR++nzJG5gd5OX1HkzNlkPrCUAzvc9kZvNT8UUx4RKrT/Jt3av2EdaKFJKnOkWPCi0Q0etjPPomTRsGxGC/KpCMIDTO4QifuI5yXm9Ap8gtt+QSc8lv95SDpWNDheRk5pCBamCGZgJnQSxWdjZBa6DTT3rbJFTtwx1mjGhereKurfft1vb3VkIC9YBvX4G2G0QMEC3X7RoOSPu8IYaJ/HKgpsuB4JfgnEAwAzoi6XYl4WYoo/dVZiVaWBS5Ne2W02O056NRZm+b7itcdIc7F+aGCkwmASzGTGS/w8f1yCzkTfonPrlA3hgLFJSzkWPEKBCeXnJrormwcSj36UAjUGN7thRxL/WirBdZuI3sRGZ9c6/UihC+9x465h/coP/fNq97nKwqzw5UaQA0dzrZauIHSSvDUMjWOLZ/LpBVBXIYDu17O8xrBa7Pe+Q6+Gaav9oganrvgqVBNZIXPb4fbefBJyU0FNQVIlwDCOSWgDWhtLGfOOZV1Fxx35RN49syEUfcSHFl7EUn+4bnlxGAAxQWUQ/ZL4FfTBEes0pFzd6geqqeg0dulgY/6TXKgBq/3HRZ5muvaY1PP/InHD93n/aVQ3g=", "X-Forefront-Antispam-Report": "CIP:216.228.112.34; CTRY:US; LANG:en; SCL:1;\n SRV:;\n IPV:NLI; SFV:NSPM; H:mail.nvidia.com; PTR:schybrid03.nvidia.com; CAT:NONE;\n SFS:(4636009)(39860400002)(396003)(136003)(376002)(346002)(36840700001)(46966006)(316002)(36906005)(2616005)(70586007)(55016002)(336012)(1076003)(426003)(2906002)(30864003)(70206006)(86362001)(8936002)(8676002)(4326008)(6286002)(107886003)(54906003)(34020700004)(82310400003)(83380400001)(26005)(7696005)(36756003)(47076005)(5660300002)(6666004)(356005)(7636003)(478600001)(186003)(16526019)(6916009)(36860700001)(82740400003);\n DIR:OUT; SFP:1101;", "X-OriginatorOrg": "Nvidia.com", "X-MS-Exchange-CrossTenant-OriginalArrivalTime": "15 Jul 2021 15:09:22.7464 (UTC)", "X-MS-Exchange-CrossTenant-Network-Message-Id": "\n 08a6430e-21f3-4284-7972-08d947a28748", "X-MS-Exchange-CrossTenant-Id": "43083d15-7273-40c1-b7db-39efd9ccc17a", "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp": "\n TenantId=43083d15-7273-40c1-b7db-39efd9ccc17a; Ip=[216.228.112.34];\n Helo=[mail.nvidia.com]", "X-MS-Exchange-CrossTenant-AuthSource": "\n CO1NAM11FT015.eop-nam11.prod.protection.outlook.com", "X-MS-Exchange-CrossTenant-AuthAs": "Anonymous", "X-MS-Exchange-CrossTenant-FromEntityHeader": "HybridOnPrem", "X-MS-Exchange-Transport-CrossTenantHeadersStamped": "BN6PR12MB1233", "Subject": "[dpdk-dev] [PATCH v7 12/16] crypto/mlx5: add enqueue and dequeue\n operations", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "From: Suanming Mou <suanmingm@nvidia.com>\n\nThe crypto operations are done with the WQE set which contains\none UMR WQE and one rdma write WQE. Most segments of the WQE\nset are initialized properly during queue setup, only limited\nsegments are initialized according to the crypto detail in the\ndatapath process.\n\nThis commit adds the enquue and dequeue operations and updates\nthe WQE set segments accordingly.\n\nSigned-off-by: Suanming Mou <suanmingm@nvidia.com>\nSigned-off-by: Matan Azrad <matan@nvidia.com>\nSigned-off-by: Michael Baum <michaelba@nvidia.com>\n---\n doc/guides/cryptodevs/features/mlx5.ini | 5 +\n drivers/crypto/mlx5/mlx5_crypto.c | 286 +++++++++++++++++++++++-\n drivers/crypto/mlx5/mlx5_crypto.h | 3 +\n 3 files changed, 290 insertions(+), 4 deletions(-)", "diff": "diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini\nindex bd757b5211..a89526add0 100644\n--- a/doc/guides/cryptodevs/features/mlx5.ini\n+++ b/doc/guides/cryptodevs/features/mlx5.ini\n@@ -6,6 +6,11 @@\n [Features]\n Symmetric crypto = Y\n HW Accelerated = Y\n+In Place SGL = Y\n+OOP SGL In SGL Out = Y\n+OOP SGL In LB Out = Y\n+OOP LB In SGL Out = Y\n+OOP LB In LB Out = Y\n Cipher multiple data units = Y\n Cipher wrapped key = Y\n \ndiff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c\nindex e5f8d96ff7..b467739a3c 100644\n--- a/drivers/crypto/mlx5/mlx5_crypto.c\n+++ b/drivers/crypto/mlx5/mlx5_crypto.c\n@@ -25,6 +25,10 @@\n \n #define MLX5_CRYPTO_FEATURE_FLAGS \\\n \t(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \\\n+\t RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \\\n+\t RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \\\n+\t RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \\\n+\t RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \\\n \t RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \\\n \t RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)\n \n@@ -297,6 +301,279 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)\n \treturn 0;\n }\n \n+static __rte_noinline uint32_t\n+mlx5_crypto_get_block_size(struct rte_crypto_op *op)\n+{\n+\tuint32_t bl = op->sym->cipher.data.length;\n+\n+\tswitch (bl) {\n+\tcase (1 << 20):\n+\t\treturn RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);\n+\tcase (1 << 12):\n+\t\treturn RTE_BE32(MLX5_BLOCK_SIZE_4096B <<\n+\t\t\t\tMLX5_BLOCK_SIZE_OFFSET);\n+\tcase (1 << 9):\n+\t\treturn RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);\n+\tdefault:\n+\t\tDRV_LOG(ERR, \"Unknown block size: %u.\", bl);\n+\t\treturn UINT32_MAX;\n+\t}\n+}\n+\n+/**\n+ * Query LKey from a packet buffer for QP. If not found, add the mempool.\n+ *\n+ * @param priv\n+ * Pointer to the priv object.\n+ * @param addr\n+ * Search key.\n+ * @param mr_ctrl\n+ * Pointer to per-queue MR control structure.\n+ * @param ol_flags\n+ * Mbuf offload features.\n+ *\n+ * @return\n+ * Searched LKey on success, UINT32_MAX on no match.\n+ */\n+static __rte_always_inline uint32_t\n+mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,\n+\t\t struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)\n+{\n+\tuint32_t lkey;\n+\n+\t/* Check generation bit to see if there's any change on existing MRs. */\n+\tif (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))\n+\t\tmlx5_mr_flush_local_cache(mr_ctrl);\n+\t/* Linear search on MR cache array. */\n+\tlkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,\n+\t\t\t\t MLX5_MR_CACHE_N, addr);\n+\tif (likely(lkey != UINT32_MAX))\n+\t\treturn lkey;\n+\t/* Take slower bottom-half on miss. */\n+\treturn mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr,\n+\t\t\t\t !!(ol_flags & EXT_ATTACHED_MBUF));\n+}\n+\n+static __rte_always_inline uint32_t\n+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,\n+\t\t struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,\n+\t\t uint32_t offset, uint32_t *remain)\n+{\n+\tuint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);\n+\tuintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);\n+\n+\tif (data_len > *remain)\n+\t\tdata_len = *remain;\n+\t*remain -= data_len;\n+\tklm->bcount = rte_cpu_to_be_32(data_len);\n+\tklm->pbuf = rte_cpu_to_be_64(addr);\n+\tklm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,\n+\t\t\t\t\tmbuf->ol_flags);\n+\treturn klm->lkey;\n+\n+}\n+\n+static __rte_always_inline uint32_t\n+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,\n+\t\t struct rte_crypto_op *op, struct rte_mbuf *mbuf,\n+\t\t struct mlx5_wqe_dseg *klm)\n+{\n+\tuint32_t remain_len = op->sym->cipher.data.length;\n+\tuint32_t nb_segs = mbuf->nb_segs;\n+\tuint32_t klm_n = 1u;\n+\n+\t/* First mbuf needs to take the cipher offset. */\n+\tif (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,\n+\t\t op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {\n+\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t\treturn 0;\n+\t}\n+\twhile (remain_len) {\n+\t\tnb_segs--;\n+\t\tmbuf = mbuf->next;\n+\t\tif (unlikely(mbuf == NULL || nb_segs == 0)) {\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;\n+\t\t\treturn 0;\n+\t\t}\n+\t\tif (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,\n+\t\t\t\t\t\t &remain_len) == UINT32_MAX)) {\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t\t\treturn 0;\n+\t\t}\n+\t\tklm_n++;\n+\t}\n+\treturn klm_n;\n+}\n+\n+static __rte_always_inline int\n+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,\n+\t\t\t struct mlx5_crypto_qp *qp,\n+\t\t\t struct rte_crypto_op *op,\n+\t\t\t struct mlx5_umr_wqe *umr)\n+{\n+\tstruct mlx5_crypto_session *sess = get_sym_session_private_data\n+\t\t\t\t(op->sym->session, mlx5_crypto_driver_id);\n+\tstruct mlx5_wqe_cseg *cseg = &umr->ctr;\n+\tstruct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;\n+\tstruct mlx5_wqe_dseg *klms = &umr->kseg[0];\n+\tstruct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)\n+\t\t\t\t RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;\n+\tuint32_t ds;\n+\tbool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;\n+\t/* Set UMR WQE. */\n+\tuint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,\n+\t\t\t\t ipl ? op->sym->m_src : op->sym->m_dst, klms);\n+\n+\tif (unlikely(klm_n == 0))\n+\t\treturn 0;\n+\tbsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;\n+\tif (unlikely(!sess->bsp_res)) {\n+\t\tbsf->bsp_res = mlx5_crypto_get_block_size(op);\n+\t\tif (unlikely(bsf->bsp_res == UINT32_MAX)) {\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;\n+\t\t\treturn 0;\n+\t\t}\n+\t} else {\n+\t\tbsf->bsp_res = sess->bsp_res;\n+\t}\n+\tbsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);\n+\tmemcpy(bsf->xts_initial_tweak,\n+\t rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);\n+\tbsf->res_dp = sess->dek_id;\n+\tmkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);\n+\tcseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);\n+\tqp->db_pi += priv->umr_wqe_stride;\n+\t/* Set RDMA_WRITE WQE. */\n+\tcseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);\n+\tklms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));\n+\tif (!ipl) {\n+\t\tklm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,\n+\t\t\t\t\t klms);\n+\t\tif (unlikely(klm_n == 0))\n+\t\t\treturn 0;\n+\t} else {\n+\t\tmemcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);\n+\t}\n+\tds = 2 + klm_n;\n+\tcseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);\n+\tcseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |\n+\t\t\t\t\t\t\tMLX5_OPCODE_RDMA_WRITE);\n+\tds = RTE_ALIGN(ds, 4);\n+\tqp->db_pi += ds >> 2;\n+\t/* Set NOP WQE if needed. */\n+\tif (priv->max_rdmar_ds > ds) {\n+\t\tcseg += ds;\n+\t\tds = priv->max_rdmar_ds - ds;\n+\t\tcseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);\n+\t\tcseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |\n+\t\t\t\t\t\t\t MLX5_OPCODE_NOP);\n+\t\tqp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */\n+\t}\n+\tqp->wqe = (uint8_t *)cseg;\n+\treturn 1;\n+}\n+\n+static __rte_always_inline void\n+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)\n+{\n+#ifdef RTE_ARCH_64\n+\t*priv->uar_addr = val;\n+#else /* !RTE_ARCH_64 */\n+\trte_spinlock_lock(&priv->uar32_sl);\n+\t*(volatile uint32_t *)priv->uar_addr = val;\n+\trte_io_wmb();\n+\t*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;\n+\trte_spinlock_unlock(&priv->uar32_sl);\n+#endif\n+}\n+\n+static uint16_t\n+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,\n+\t\t\t uint16_t nb_ops)\n+{\n+\tstruct mlx5_crypto_qp *qp = queue_pair;\n+\tstruct mlx5_crypto_priv *priv = qp->priv;\n+\tstruct mlx5_umr_wqe *umr;\n+\tstruct rte_crypto_op *op;\n+\tuint16_t mask = qp->entries_n - 1;\n+\tuint16_t remain = qp->entries_n - (qp->pi - qp->ci);\n+\n+\tif (remain < nb_ops)\n+\t\tnb_ops = remain;\n+\telse\n+\t\tremain = nb_ops;\n+\tif (unlikely(remain == 0))\n+\t\treturn 0;\n+\tdo {\n+\t\top = *ops++;\n+\t\tumr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);\n+\t\tif (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {\n+\t\t\tif (remain != nb_ops)\n+\t\t\t\tbreak;\n+\t\t\treturn 0;\n+\t\t}\n+\t\tqp->ops[qp->pi] = op;\n+\t\tqp->pi = (qp->pi + 1) & mask;\n+\t} while (--remain);\n+\trte_io_wmb();\n+\tqp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);\n+\trte_wmb();\n+\tmlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);\n+\trte_wmb();\n+\treturn nb_ops;\n+}\n+\n+static __rte_noinline void\n+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)\n+{\n+\tconst uint32_t idx = qp->ci & (qp->entries_n - 1);\n+\tvolatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)\n+\t\t\t\t\t\t\t&qp->cq_obj.cqes[idx];\n+\n+\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\tDRV_LOG(ERR, \"CQE ERR:%x.\\n\", rte_be_to_cpu_32(cqe->syndrome));\n+}\n+\n+static uint16_t\n+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,\n+\t\t\t uint16_t nb_ops)\n+{\n+\tstruct mlx5_crypto_qp *qp = queue_pair;\n+\tvolatile struct mlx5_cqe *restrict cqe;\n+\tstruct rte_crypto_op *restrict op;\n+\tconst unsigned int cq_size = qp->entries_n;\n+\tconst unsigned int mask = cq_size - 1;\n+\tuint32_t idx;\n+\tuint32_t next_idx = qp->ci & mask;\n+\tconst uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);\n+\tuint16_t i = 0;\n+\tint ret;\n+\n+\tif (unlikely(max == 0))\n+\t\treturn 0;\n+\tdo {\n+\t\tidx = next_idx;\n+\t\tnext_idx = (qp->ci + 1) & mask;\n+\t\top = qp->ops[idx];\n+\t\tcqe = &qp->cq_obj.cqes[idx];\n+\t\tret = check_cqe(cqe, cq_size, qp->ci);\n+\t\trte_io_rmb();\n+\t\tif (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {\n+\t\t\tif (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))\n+\t\t\t\tmlx5_crypto_cqe_err_handle(qp, op);\n+\t\t\tbreak;\n+\t\t}\n+\t\top->status = RTE_CRYPTO_OP_STATUS_SUCCESS;\n+\t\tops[i++] = op;\n+\t\tqp->ci++;\n+\t} while (i < max);\n+\tif (likely(i != 0)) {\n+\t\trte_io_wmb();\n+\t\tqp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);\n+\t}\n+\treturn i;\n+}\n+\n static void\n mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)\n {\n@@ -521,8 +798,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)\n \tif (mlx5_crypto_pd_create(priv) != 0)\n \t\treturn -1;\n \tpriv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);\n-\tif (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==\n-\t NULL) {\n+\tif (priv->uar)\n+\t\tpriv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);\n+\tif (priv->uar == NULL || priv->uar_addr == NULL) {\n \t\trte_errno = errno;\n \t\tclaim_zero(mlx5_glue->dealloc_pd(priv->pd));\n \t\tDRV_LOG(ERR, \"Failed to allocate UAR.\");\n@@ -752,8 +1030,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,\n \tDRV_LOG(INFO,\n \t\t\"Crypto device %s was created successfully.\", ibv->name);\n \tcrypto_dev->dev_ops = &mlx5_crypto_ops;\n-\tcrypto_dev->dequeue_burst = NULL;\n-\tcrypto_dev->enqueue_burst = NULL;\n+\tcrypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;\n+\tcrypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;\n \tcrypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;\n \tcrypto_dev->driver_id = mlx5_crypto_driver_id;\n \tpriv = crypto_dev->data->dev_private;\ndiff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h\nindex d9b1ff8e99..1350513b9e 100644\n--- a/drivers/crypto/mlx5/mlx5_crypto.h\n+++ b/drivers/crypto/mlx5/mlx5_crypto.h\n@@ -36,6 +36,9 @@ struct mlx5_crypto_priv {\n \tuint16_t umr_wqe_size;\n \tuint16_t umr_wqe_stride;\n \tuint16_t max_rdmar_ds;\n+#ifndef RTE_ARCH_64\n+\trte_spinlock_t uar32_sl;\n+#endif /* RTE_ARCH_64 */\n };\n \n struct mlx5_crypto_qp {\n", "prefixes": [ "v7", "12/16" ] }