Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/139558/?format=api
{ "id": 139558, "url": "https://patches.dpdk.org/api/patches/139558/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20240419195310.21432-6-andrew.boyer@amd.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20240419195310.21432-6-andrew.boyer@amd.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20240419195310.21432-6-andrew.boyer@amd.com", "date": "2024-04-19T19:53:09", "name": "[5/6] crypto/ionic: add datapath and capabilities support", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "8e0f76b20261d0efae0f96c4217adaa02df399f7", "submitter": { "id": 2861, "url": "https://patches.dpdk.org/api/people/2861/?format=api", "name": "Andrew Boyer", "email": "Andrew.Boyer@amd.com" }, "delegate": { "id": 6690, "url": "https://patches.dpdk.org/api/users/6690/?format=api", "username": "akhil", "first_name": "akhil", "last_name": "goyal", "email": "gakhil@marvell.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20240419195310.21432-6-andrew.boyer@amd.com/mbox/", "series": [ { "id": 31795, "url": "https://patches.dpdk.org/api/series/31795/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=31795", "date": "2024-04-19T19:53:04", "name": "crypto/ionic: introduce AMD Pensando ionic crypto driver", "version": 1, "mbox": "https://patches.dpdk.org/series/31795/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/139558/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/139558/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 83F2C43EB5;\n\tFri, 19 Apr 2024 21:54:33 +0200 (CEST)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 66C3840E09;\n\tFri, 19 Apr 2024 21:54:00 +0200 (CEST)", "from NAM04-BN8-obe.outbound.protection.outlook.com\n (mail-bn8nam04on2087.outbound.protection.outlook.com [40.107.100.87])\n by mails.dpdk.org (Postfix) with ESMTP id B23D240DDE\n for <dev@dpdk.org>; Fri, 19 Apr 2024 21:53:58 +0200 (CEST)", "from SA0PR11CA0062.namprd11.prod.outlook.com (2603:10b6:806:d2::7)\n by BY5PR12MB4225.namprd12.prod.outlook.com (2603:10b6:a03:211::20) with\n Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.7452.50; Fri, 19 Apr\n 2024 19:53:54 +0000", "from SA2PEPF00001504.namprd04.prod.outlook.com\n (2603:10b6:806:d2:cafe::57) by SA0PR11CA0062.outlook.office365.com\n (2603:10b6:806:d2::7) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.7495.31 via Frontend\n Transport; Fri, 19 Apr 2024 19:53:53 +0000", "from SATLEXMB04.amd.com (165.204.84.17) by\n SA2PEPF00001504.mail.protection.outlook.com (10.167.242.36) with Microsoft\n SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id\n 15.20.7452.22 via Frontend Transport; Fri, 19 Apr 2024 19:53:52 +0000", "from driver-dev1.pensando.io (10.180.168.240) by SATLEXMB04.amd.com\n (10.181.40.145) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2507.35; Fri, 19 Apr\n 2024 14:53:51 -0500" ], "ARC-Seal": "i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none;\n b=UTOosoaKXzO+dgmvhvQAMqpML+Ja1ZZPY5PWbrP6A5VFQyCE+enQ1b8qzAW8MPFdSIfJfgz4hdhqm/1cXXo0QmXWaYXCltGOtvtDuEn9ezqzkpn4UZ3u208oVCpi799Cxzu4j9uyzTKD+9l+SBx5tWg6CAPNUGWKCeO4QU/9rF19z5VyUkhbqR5H5HDXqbhPKDyn57y5tPdBSszNi7hnU61wbEm2Y3EN2fTahIv/3Wlt2kKz+Zsv2cIeZhyT88UnXttk7zWVL/f636cfVXow10DhQ9pSSjdBLB/HFPIKsxpXmyx8eyr5h+U3uRGZtDGG3hCaupBfYodZocewzjbaRQ==", "ARC-Message-Signature": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com;\n s=arcselector9901;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1;\n bh=49Mc/TpuYdRdUu0FvkLQwlB8NcLhWGS4D/vT7JSp/B0=;\n b=XTypQpP4A6s4pP6NznG08NGhbu06iAH8ydXKULqlBvTbWQETN96833skekSzPrJOle+52H9G6VGtuwwAFUOJtdjzlb7844EkDenMup3anbgPCArfyVePDk9GVGqyuTMX2NpvDqKsvNgysZr7GALqFTRnpktTWSHAQUkw5S2/9LrhCCT8qqQAnAfxw6TDyznUdzKASLDDUOwnAVNFlvSI7TgtrZk4w5z6aY0mPNm1qP2irVpGtqlrReMMYDRDf/lio32BaxPoZgTHz0mzXkFz3fW7PgKQdXbOOpW0pBS27BDg5AhpD5VOV4ORInuFthEWKLqMI3qDbvo93/cCC5B4gw==", "ARC-Authentication-Results": "i=1; mx.microsoft.com 1; spf=pass (sender ip is\n 165.204.84.17) smtp.rcpttodomain=dpdk.org smtp.mailfrom=amd.com; dmarc=pass\n (p=quarantine sp=quarantine pct=100) action=none header.from=amd.com;\n dkim=none (message not signed); arc=none (0)", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=amd.com; s=selector1;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=49Mc/TpuYdRdUu0FvkLQwlB8NcLhWGS4D/vT7JSp/B0=;\n b=tG675WwCSfqhSpfO4DqYcY7nmFMyzu5A6ZCo6RCPr9VZdld5083KkCOXHy6aEVd0sCT7xM7SdYMc3pROXRUXocf2l0QxUn5cCt+F+ezDvus6etQ9qWLdvORwOwTaJH7C77iHL5ivUildvPX8oYFoFsDYbkl/hN2PSZN6pRfjMP0=", "X-MS-Exchange-Authentication-Results": "spf=pass (sender IP is 165.204.84.17)\n smtp.mailfrom=amd.com; dkim=none (message not signed)\n header.d=none;dmarc=pass action=none header.from=amd.com;", "Received-SPF": "Pass (protection.outlook.com: domain of amd.com designates\n 165.204.84.17 as permitted sender) receiver=protection.outlook.com;\n client-ip=165.204.84.17; helo=SATLEXMB04.amd.com; pr=C", "From": "Andrew Boyer <andrew.boyer@amd.com>", "To": "<dev@dpdk.org>", "CC": "Akhil Goyal <gakhil@marvell.com>, Andrew Boyer <andrew.boyer@amd.com>", "Subject": "[PATCH 5/6] crypto/ionic: add datapath and capabilities support", "Date": "Fri, 19 Apr 2024 12:53:09 -0700", "Message-ID": "<20240419195310.21432-6-andrew.boyer@amd.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20240419195310.21432-1-andrew.boyer@amd.com>", "References": "<20240419195310.21432-1-andrew.boyer@amd.com>", "MIME-Version": "1.0", "Content-Type": "text/plain", "X-Originating-IP": "[10.180.168.240]", "X-ClientProxiedBy": "SATLEXMB04.amd.com (10.181.40.145) To SATLEXMB04.amd.com\n (10.181.40.145)", "X-EOPAttributedMessage": "0", "X-MS-PublicTrafficType": "Email", "X-MS-TrafficTypeDiagnostic": "SA2PEPF00001504:EE_|BY5PR12MB4225:EE_", "X-MS-Office365-Filtering-Correlation-Id": "69dcbeea-9aa3-4fee-472f-08dc60aa7097", "X-MS-Exchange-SenderADCheck": "1", "X-MS-Exchange-AntiSpam-Relay": "0", "X-Microsoft-Antispam": "BCL:0;", "X-Microsoft-Antispam-Message-Info": "\n g+e20lQBQMZnitfS/kU2bVCdg3Wk1QtChEPcAWrrJ5OpT3SvhVPEGYRHR0oQDR9DZp/04C36g50nT36Us+C4swNZ5IfdPYyIxZZf+ULLFsbHlJ0kTsMzi4YfvnszOeObehQw8/PxPwnhzD4/nLmplc0KA7tgYYS3YbkxsjsTYQvVmJ0KMH4dpHFmyvgKQ3mC8eFN07RkeDRN1stbAFcmKoYWDxSTyklaMTtgeS3gHpEO9bahm4HmF7+80FVJHZzxnvwOhAWzXFfFA9zFe6G3HEhrvBaTafeJlur7JHKPATOAijdWR8Tg4o8iwG+TQr8+x8YmG5Qap92Ss9o+EX0kgPKlAeOPBv/j7WN5vLOq+ZMhNgmz2bNDa1Mg2kX2MCRXH07MCm+4k2/Aqvuv2emlRo9nIHgXQMmvftUxxxNo4xGNow/baCQha9zkFW3aTxW4nQXqiQxzNM+JmKfhWel65641N6/AZOxpQRVGetvy8sZEyraCeZrZdO2mwaJnwzUjxOahsEduuyvPoXNpsl5Wg6FySXp8b1ReyBpAv8KrINSsQ383W1Ze/VZZQArv6CmPzN7XfxUhGDSfwsLxD5n+G4ObVF+Wxnr7QuXqxHhVkvZ/Avsunek0QbrhasprWPK7TWlb/CtykLB5QoDAbC42UoB0MMuXcDPwKjucY6JcMnRH2YBx5RvA3eXiJ5o1oZd6obSmmuKGlvW0zMVL7Rv4vSnDCGEgPKzyiEWOguThw1Mf7NFAiyPpZJZ68YiEfy+L/2mBKcZEgAgqQ4K1n76G/3X8o5l3PCFqx7+R5ePwFjgUwhbjZTG4ngrNGGtBP5mHxLaAAc6SySIIrEzzqaRpkURuu8VuKITodLkH7z4tj3IQO/ng1/tu3wWM8dN57pwHmmt/yEEiyQqB6A8/7XM3nxPfdt4jk/jcYUp7ecnXtDWJv6xhcEC/FXDl/fkBhUP/GEsU9QjrVN/ATWs9gcvt6j5MnKurj2sDPcVcgy32/27LM1TUEq4D/I0sfdyeV8EBwpWPtXDa7aHxE6ju1RsF2w0fgimP7FN+IMqVfw/bAee5aFmBeiYnniUzNI845uH4Ja/4xQV26exXIRRpfp2y9FXs2EN0wrSwh9elsenZLchwZLZ6fK9es4Ch8Y83AUTvVdTtYmyFjVrBo7WDARbfTcAtRNO0ATVc013qJmxCkDfYo8ozXgAMqMUvi+KervRIfcFjun7KqoZh8UcU2cYKki9Ic/D5ZmcnXxPPm30K1KoaVJzGkHLHn2Qg/BfxpgT4dzkoZVi8UJFsfmRRGsWJdp4ZEJx8TRi/z1KloPNuJlDkfmeUS+qX4S9E/SOTuyUog8DcMovdast0tgSIiXfRSwzzc+KcZFYpUuCQlebRC9Y=", "X-Forefront-Antispam-Report": "CIP:165.204.84.17; CTRY:US; LANG:en; SCL:1; SRV:;\n IPV:CAL; SFV:NSPM; H:SATLEXMB04.amd.com; PTR:InfoDomainNonexistent; CAT:NONE;\n SFS:(13230031)(1800799015)(36860700004)(82310400014)(376005); DIR:OUT;\n SFP:1101;", "X-OriginatorOrg": "amd.com", "X-MS-Exchange-CrossTenant-OriginalArrivalTime": "19 Apr 2024 19:53:52.8811 (UTC)", "X-MS-Exchange-CrossTenant-Network-Message-Id": "\n 69dcbeea-9aa3-4fee-472f-08dc60aa7097", "X-MS-Exchange-CrossTenant-Id": "3dd8961f-e488-4e60-8e11-a82d994e183d", "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp": "\n TenantId=3dd8961f-e488-4e60-8e11-a82d994e183d; Ip=[165.204.84.17];\n Helo=[SATLEXMB04.amd.com]", "X-MS-Exchange-CrossTenant-AuthSource": "\n SA2PEPF00001504.namprd04.prod.outlook.com", "X-MS-Exchange-CrossTenant-AuthAs": "Anonymous", "X-MS-Exchange-CrossTenant-FromEntityHeader": "HybridOnPrem", "X-MS-Exchange-Transport-CrossTenantHeadersStamped": "BY5PR12MB4225", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "This defines the main datapath and reports the device\ncapabilities to the stack.\n\nSigned-off-by: Andrew Boyer <andrew.boyer@amd.com>\n---\n drivers/crypto/ionic/ionic_crypto.h | 62 +++\n drivers/crypto/ionic/ionic_crypto_caps.c | 55 ++\n drivers/crypto/ionic/ionic_crypto_main.c | 417 +++++++++++++++-\n drivers/crypto/ionic/ionic_crypto_ops.c | 606 +++++++++++++++++++++++\n drivers/crypto/ionic/meson.build | 2 +\n 5 files changed, 1140 insertions(+), 2 deletions(-)\n create mode 100644 drivers/crypto/ionic/ionic_crypto_caps.c\n create mode 100644 drivers/crypto/ionic/ionic_crypto_ops.c", "diff": "diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h\nindex d048f7aa51..db87ea0490 100644\n--- a/drivers/crypto/ionic/ionic_crypto.h\n+++ b/drivers/crypto/ionic/ionic_crypto.h\n@@ -37,6 +37,8 @@ extern int iocpt_logtype;\n \n #define IOCPT_PRINT_CALL() IOCPT_PRINT(DEBUG, \" >>\")\n \n+const struct rte_cryptodev_capabilities *iocpt_get_caps(uint64_t flags);\n+\n static inline void iocpt_struct_size_checks(void)\n {\n \tRTE_BUILD_BUG_ON(sizeof(struct ionic_doorbell) != 8);\n@@ -163,6 +165,50 @@ struct iocpt_admin_q {\n \tuint16_t flags;\n };\n \n+struct iocpt_crypto_q {\n+\t/* cacheline0, cacheline1 */\n+\tIOCPT_COMMON_FIELDS;\n+\n+\t/* cacheline2 */\n+\tuint64_t last_wdog_cycles;\n+\tuint16_t flags;\n+\n+\t/* cacheline3 */\n+\tstruct rte_cryptodev_stats stats;\n+\n+\tuint64_t enqueued_wdogs;\n+\tuint64_t dequeued_wdogs;\n+\tuint8_t wdog_iv[IOCPT_Q_WDOG_IV_LEN];\n+\tuint8_t wdog_pld[IOCPT_Q_WDOG_PLD_LEN];\n+\tuint8_t wdog_tag[IOCPT_Q_WDOG_TAG_LEN];\n+};\n+\n+#define IOCPT_S_F_INITED\tBIT(0)\n+\n+struct iocpt_session_priv {\n+\tstruct iocpt_dev *dev;\n+\n+\tuint32_t index;\n+\n+\tuint16_t iv_offset;\n+\tuint16_t iv_length;\n+\tuint16_t digest_length;\n+\tuint16_t aad_length;\n+\n+\tuint8_t flags;\n+\tuint8_t op;\n+\tuint8_t type;\n+\n+\tuint16_t key_len;\n+\tuint8_t key[IOCPT_SESS_KEY_LEN_MAX_SYMM];\n+};\n+\n+static inline uint32_t\n+iocpt_session_size(void)\n+{\n+\treturn sizeof(struct iocpt_session_priv);\n+}\n+\n #define IOCPT_DEV_F_INITED\t\tBIT(0)\n #define IOCPT_DEV_F_UP\t\t\tBIT(1)\n #define IOCPT_DEV_F_FW_RESET\t\tBIT(2)\n@@ -194,6 +240,7 @@ struct iocpt_dev {\n \trte_spinlock_t adminq_service_lock;\n \n \tstruct iocpt_admin_q *adminq;\n+\tstruct iocpt_crypto_q **cryptoqs;\n \n \tstruct rte_bitmap *sess_bm;\t/* SET bit indicates index is free */\n \n@@ -242,6 +289,9 @@ int iocpt_probe(void *bus_dev, struct rte_device *rte_dev,\n int iocpt_remove(struct rte_device *rte_dev);\n \n void iocpt_configure(struct iocpt_dev *dev);\n+int iocpt_assign_ops(struct rte_cryptodev *cdev);\n+int iocpt_start(struct iocpt_dev *dev);\n+void iocpt_stop(struct iocpt_dev *dev);\n void iocpt_deinit(struct iocpt_dev *dev);\n \n int iocpt_dev_identify(struct iocpt_dev *dev);\n@@ -251,6 +301,14 @@ void iocpt_dev_reset(struct iocpt_dev *dev);\n \n int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx);\n \n+int iocpt_cryptoq_alloc(struct iocpt_dev *dev, uint32_t socket_id,\n+\tuint32_t index, uint16_t ndescs);\n+void iocpt_cryptoq_free(struct iocpt_crypto_q *cptq);\n+\n+int iocpt_session_init(struct iocpt_session_priv *priv);\n+int iocpt_session_update(struct iocpt_session_priv *priv);\n+void iocpt_session_deinit(struct iocpt_session_priv *priv);\n+\n struct ionic_doorbell __iomem *iocpt_db_map(struct iocpt_dev *dev,\n \tstruct iocpt_queue *q);\n \n@@ -259,6 +317,10 @@ typedef bool (*iocpt_cq_cb)(struct iocpt_cq *cq, uint16_t cq_desc_index,\n uint32_t iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,\n \tiocpt_cq_cb cb, void *cb_arg);\n \n+void iocpt_get_stats(const struct iocpt_dev *dev,\n+\tstruct rte_cryptodev_stats *stats);\n+void iocpt_reset_stats(struct iocpt_dev *dev);\n+\n static inline uint16_t\n iocpt_q_space_avail(struct iocpt_queue *q)\n {\ndiff --git a/drivers/crypto/ionic/ionic_crypto_caps.c b/drivers/crypto/ionic/ionic_crypto_caps.c\nnew file mode 100644\nindex 0000000000..da5a69be3d\n--- /dev/null\n+++ b/drivers/crypto/ionic/ionic_crypto_caps.c\n@@ -0,0 +1,55 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2021-2024 Advanced Micro Devices, Inc.\n+ */\n+\n+#include <rte_cryptodev.h>\n+\n+#include \"ionic_crypto.h\"\n+\n+static const struct rte_cryptodev_capabilities iocpt_sym_caps[] = {\n+\t{\t/* AES GCM */\n+\t\t.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,\n+\t\t{.sym = {\n+\t\t\t.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,\n+\t\t\t{.aead = {\n+\t\t\t\t.algo = RTE_CRYPTO_AEAD_AES_GCM,\n+\t\t\t\t.block_size = 16,\n+\t\t\t\t.key_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 32,\n+\t\t\t\t\t.increment = 16\n+\t\t\t\t},\n+\t\t\t\t.digest_size = {\n+\t\t\t\t\t.min = 16,\n+\t\t\t\t\t.max = 16,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t},\n+\t\t\t\t.aad_size = {\n+\t\t\t\t\t.min = 0,\n+\t\t\t\t\t.max = 1024,\n+\t\t\t\t\t.increment = 1\n+\t\t\t\t},\n+\t\t\t\t.iv_size = {\n+\t\t\t\t\t.min = 12,\n+\t\t\t\t\t.max = 12,\n+\t\t\t\t\t.increment = 0\n+\t\t\t\t}\n+\t\t\t}, }\n+\t\t}, }\n+\t},\n+\tRTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()\n+};\n+\n+static const struct rte_cryptodev_capabilities iocpt_asym_caps[] = {\n+\t/* None */\n+\tRTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()\n+};\n+\n+const struct rte_cryptodev_capabilities *\n+iocpt_get_caps(uint64_t flags)\n+{\n+\tif (flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO)\n+\t\treturn iocpt_asym_caps;\n+\telse\n+\t\treturn iocpt_sym_caps;\n+}\ndiff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c\nindex 84aff65f22..543f797d51 100644\n--- a/drivers/crypto/ionic/ionic_crypto_main.c\n+++ b/drivers/crypto/ionic/ionic_crypto_main.c\n@@ -31,6 +31,15 @@ iocpt_cq_init(struct iocpt_cq *cq, uint16_t num_descs)\n \treturn 0;\n }\n \n+static void\n+iocpt_cq_reset(struct iocpt_cq *cq)\n+{\n+\tcq->tail_idx = 0;\n+\tcq->done_color = 1;\n+\n+\tmemset(cq->base, 0, sizeof(struct iocpt_nop_comp) * cq->num_descs);\n+}\n+\n static void\n iocpt_cq_map(struct iocpt_cq *cq, void *base, rte_iova_t base_pa)\n {\n@@ -91,6 +100,13 @@ iocpt_q_init(struct iocpt_queue *q, uint8_t type, uint32_t index,\n \treturn 0;\n }\n \n+static void\n+iocpt_q_reset(struct iocpt_queue *q)\n+{\n+\tq->head_idx = 0;\n+\tq->tail_idx = 0;\n+}\n+\n static void\n iocpt_q_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)\n {\n@@ -114,6 +130,178 @@ iocpt_q_free(struct iocpt_queue *q)\n \t}\n }\n \n+static void\n+iocpt_get_abs_stats(const struct iocpt_dev *dev,\n+\t\tstruct rte_cryptodev_stats *stats)\n+{\n+\tuint32_t i;\n+\n+\tmemset(stats, 0, sizeof(*stats));\n+\n+\t/* Sum up the per-queue stats counters */\n+\tfor (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {\n+\t\tstruct rte_cryptodev_stats *q_stats = &dev->cryptoqs[i]->stats;\n+\n+\t\tstats->enqueued_count += q_stats->enqueued_count;\n+\t\tstats->dequeued_count += q_stats->dequeued_count;\n+\t\tstats->enqueue_err_count += q_stats->enqueue_err_count;\n+\t\tstats->dequeue_err_count += q_stats->dequeue_err_count;\n+\t}\n+}\n+\n+void\n+iocpt_get_stats(const struct iocpt_dev *dev, struct rte_cryptodev_stats *stats)\n+{\n+\t/* Retrieve the new absolute stats values */\n+\tiocpt_get_abs_stats(dev, stats);\n+\n+\t/* Subtract the base stats values to get relative values */\n+\tstats->enqueued_count -= dev->stats_base.enqueued_count;\n+\tstats->dequeued_count -= dev->stats_base.dequeued_count;\n+\tstats->enqueue_err_count -= dev->stats_base.enqueue_err_count;\n+\tstats->dequeue_err_count -= dev->stats_base.dequeue_err_count;\n+}\n+\n+void\n+iocpt_reset_stats(struct iocpt_dev *dev)\n+{\n+\tuint32_t i;\n+\n+\t/* Erase the per-queue stats counters */\n+\tfor (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++)\n+\t\tmemset(&dev->cryptoqs[i]->stats, 0,\n+\t\t\tsizeof(dev->cryptoqs[i]->stats));\n+\n+\t/* Update the base stats values */\n+\tiocpt_get_abs_stats(dev, &dev->stats_base);\n+}\n+\n+static int\n+iocpt_session_write(struct iocpt_session_priv *priv,\n+\t\t enum iocpt_sess_control_oper oper)\n+{\n+\tstruct iocpt_dev *dev = priv->dev;\n+\tstruct iocpt_admin_ctx ctx = {\n+\t\t.pending_work = true,\n+\t\t.cmd.sess_control = {\n+\t\t\t.opcode = IOCPT_CMD_SESS_CONTROL,\n+\t\t\t.type = priv->type,\n+\t\t\t.oper = oper,\n+\t\t\t.index = rte_cpu_to_le_32(priv->index),\n+\t\t\t.key_len = rte_cpu_to_le_16(priv->key_len),\n+\t\t\t.key_seg_len = (uint8_t)RTE_MIN(priv->key_len,\n+\t\t\t\t\t\tIOCPT_SESS_KEY_SEG_LEN),\n+\t\t},\n+\t};\n+\tstruct iocpt_sess_control_cmd *cmd = &ctx.cmd.sess_control;\n+\tuint16_t key_offset;\n+\tuint8_t key_segs, seg;\n+\tint err;\n+\n+\tkey_segs = ((priv->key_len - 1) >> IOCPT_SESS_KEY_SEG_SHFT) + 1;\n+\n+\tfor (seg = 0; seg < key_segs; seg++) {\n+\t\tctx.pending_work = true;\n+\n+\t\tkey_offset = seg * cmd->key_seg_len;\n+\t\tmemcpy(cmd->key, &priv->key[key_offset],\n+\t\t\tIOCPT_SESS_KEY_SEG_LEN);\n+\t\tcmd->key_seg_idx = seg;\n+\n+\t\t/* Mark final segment */\n+\t\tif (seg + 1 == key_segs)\n+\t\t\tcmd->flags |= rte_cpu_to_le_16(IOCPT_SCTL_F_END);\n+\n+\t\terr = iocpt_adminq_post_wait(dev, &ctx);\n+\t\tif (err != 0)\n+\t\t\treturn err;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int\n+iocpt_session_wdog(struct iocpt_dev *dev)\n+{\n+\tstruct iocpt_session_priv priv = {\n+\t\t.dev = dev,\n+\t\t.index = IOCPT_Q_WDOG_SESS_IDX,\n+\t\t.type = IOCPT_SESS_AEAD_AES_GCM,\n+\t\t.key_len = IOCPT_Q_WDOG_KEY_LEN,\n+\t};\n+\n+\t/* Reserve session 0 for queue watchdog */\n+\trte_bitmap_clear(dev->sess_bm, IOCPT_Q_WDOG_SESS_IDX);\n+\n+\treturn iocpt_session_write(&priv, IOCPT_SESS_INIT);\n+}\n+\n+int\n+iocpt_session_init(struct iocpt_session_priv *priv)\n+{\n+\tstruct iocpt_dev *dev = priv->dev;\n+\tuint64_t bm_slab = 0;\n+\tuint32_t bm_pos = 0;\n+\tint err = 0;\n+\n+\trte_spinlock_lock(&dev->adminq_lock);\n+\n+\tif (rte_bitmap_scan(dev->sess_bm, &bm_pos, &bm_slab) > 0) {\n+\t\tpriv->index = bm_pos + rte_ctz64(bm_slab);\n+\t\trte_bitmap_clear(dev->sess_bm, priv->index);\n+\t} else\n+\t\terr = -ENOSPC;\n+\n+\trte_spinlock_unlock(&dev->adminq_lock);\n+\n+\tif (err != 0) {\n+\t\tIOCPT_PRINT(ERR, \"session index space exhausted\");\n+\t\treturn err;\n+\t}\n+\n+\terr = iocpt_session_write(priv, IOCPT_SESS_INIT);\n+\tif (err != 0) {\n+\t\trte_spinlock_lock(&dev->adminq_lock);\n+\t\trte_bitmap_set(dev->sess_bm, priv->index);\n+\t\trte_spinlock_unlock(&dev->adminq_lock);\n+\t\treturn err;\n+\t}\n+\n+\tpriv->flags |= IOCPT_S_F_INITED;\n+\n+\treturn 0;\n+}\n+\n+int\n+iocpt_session_update(struct iocpt_session_priv *priv)\n+{\n+\treturn iocpt_session_write(priv, IOCPT_SESS_UPDATE_KEY);\n+}\n+\n+void\n+iocpt_session_deinit(struct iocpt_session_priv *priv)\n+{\n+\tstruct iocpt_dev *dev = priv->dev;\n+\tstruct iocpt_admin_ctx ctx = {\n+\t\t.pending_work = true,\n+\t\t.cmd.sess_control = {\n+\t\t\t.opcode = IOCPT_CMD_SESS_CONTROL,\n+\t\t\t.type = priv->type,\n+\t\t\t.oper = IOCPT_SESS_DISABLE,\n+\t\t\t.index = rte_cpu_to_le_32(priv->index),\n+\t\t\t.key_len = rte_cpu_to_le_16(priv->key_len),\n+\t\t},\n+\t};\n+\n+\t(void)iocpt_adminq_post_wait(dev, &ctx);\n+\n+\trte_spinlock_lock(&dev->adminq_lock);\n+\trte_bitmap_set(dev->sess_bm, priv->index);\n+\trte_spinlock_unlock(&dev->adminq_lock);\n+\n+\tpriv->flags &= ~IOCPT_S_F_INITED;\n+}\n+\n static const struct rte_memzone *\n iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size,\n \t\t\tunsigned int align, int socket_id)\n@@ -240,12 +428,157 @@ iocpt_commonq_alloc(struct iocpt_dev *dev,\n \treturn err;\n }\n \n+int\n+iocpt_cryptoq_alloc(struct iocpt_dev *dev, uint32_t socket_id, uint32_t index,\n+\t\tuint16_t num_descs)\n+{\n+\tstruct iocpt_crypto_q *cptq;\n+\tuint16_t flags = 0;\n+\tint err;\n+\n+\t/* CryptoQ always supports scatter-gather */\n+\tflags |= IOCPT_Q_F_SG;\n+\n+\tIOCPT_PRINT(DEBUG, \"cptq %u num_descs %u num_segs %u\",\n+\t\tindex, num_descs, 1);\n+\n+\terr = iocpt_commonq_alloc(dev,\n+\t\tIOCPT_QTYPE_CRYPTOQ,\n+\t\tsizeof(struct iocpt_crypto_q),\n+\t\tsocket_id,\n+\t\tindex,\n+\t\t\"crypto\",\n+\t\tflags,\n+\t\tnum_descs,\n+\t\t1,\n+\t\tsizeof(struct iocpt_crypto_desc),\n+\t\tsizeof(struct iocpt_crypto_comp),\n+\t\tsizeof(struct iocpt_crypto_sg_desc),\n+\t\t(struct iocpt_common_q **)&cptq);\n+\tif (err != 0)\n+\t\treturn err;\n+\n+\tcptq->flags = flags;\n+\n+\tdev->cryptoqs[index] = cptq;\n+\n+\treturn 0;\n+}\n+\n struct ionic_doorbell *\n iocpt_db_map(struct iocpt_dev *dev, struct iocpt_queue *q)\n {\n \treturn dev->db_pages + q->hw_type;\n }\n \n+static int\n+iocpt_cryptoq_init(struct iocpt_crypto_q *cptq)\n+{\n+\tstruct iocpt_queue *q = &cptq->q;\n+\tstruct iocpt_dev *dev = cptq->dev;\n+\tstruct iocpt_cq *cq = &cptq->cq;\n+\tstruct iocpt_admin_ctx ctx = {\n+\t\t.pending_work = true,\n+\t\t.cmd.q_init = {\n+\t\t\t.opcode = IOCPT_CMD_Q_INIT,\n+\t\t\t.type = IOCPT_QTYPE_CRYPTOQ,\n+\t\t\t.ver = dev->qtype_info[IOCPT_QTYPE_CRYPTOQ].version,\n+\t\t\t.index = rte_cpu_to_le_32(q->index),\n+\t\t\t.flags = rte_cpu_to_le_16(IOCPT_QINIT_F_ENA |\n+\t\t\t\t\t\tIOCPT_QINIT_F_SG),\n+\t\t\t.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),\n+\t\t\t.ring_size = rte_log2_u32(q->num_descs),\n+\t\t\t.ring_base = rte_cpu_to_le_64(q->base_pa),\n+\t\t\t.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),\n+\t\t\t.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),\n+\t\t},\n+\t};\n+\tint err;\n+\n+\tIOCPT_PRINT(DEBUG, \"cptq_init.index %d\", q->index);\n+\tIOCPT_PRINT(DEBUG, \"cptq_init.ring_base %#jx\", q->base_pa);\n+\tIOCPT_PRINT(DEBUG, \"cptq_init.ring_size %d\",\n+\t\tctx.cmd.q_init.ring_size);\n+\tIOCPT_PRINT(DEBUG, \"cptq_init.ver %u\", ctx.cmd.q_init.ver);\n+\n+\tiocpt_q_reset(q);\n+\tiocpt_cq_reset(cq);\n+\n+\terr = iocpt_adminq_post_wait(dev, &ctx);\n+\tif (err != 0)\n+\t\treturn err;\n+\n+\tq->hw_type = ctx.comp.q_init.hw_type;\n+\tq->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index);\n+\tq->db = iocpt_db_map(dev, q);\n+\n+\tIOCPT_PRINT(DEBUG, \"cptq->hw_type %d\", q->hw_type);\n+\tIOCPT_PRINT(DEBUG, \"cptq->hw_index %d\", q->hw_index);\n+\tIOCPT_PRINT(DEBUG, \"cptq->db %p\", q->db);\n+\n+\tcptq->flags |= IOCPT_Q_F_INITED;\n+\n+\treturn 0;\n+}\n+\n+static void\n+iocpt_cryptoq_deinit(struct iocpt_crypto_q *cptq)\n+{\n+\tstruct iocpt_dev *dev = cptq->dev;\n+\tstruct iocpt_admin_ctx ctx = {\n+\t\t.pending_work = true,\n+\t\t.cmd.q_control = {\n+\t\t\t.opcode = IOCPT_CMD_Q_CONTROL,\n+\t\t\t.type = IOCPT_QTYPE_CRYPTOQ,\n+\t\t\t.index = rte_cpu_to_le_32(cptq->q.index),\n+\t\t\t.oper = IOCPT_Q_DISABLE,\n+\t\t},\n+\t};\n+\tunsigned long sleep_usec = 100UL * 1000;\n+\tuint32_t sleep_cnt, sleep_max = IOCPT_CRYPTOQ_WAIT;\n+\tint err;\n+\n+\tfor (sleep_cnt = 0; sleep_cnt < sleep_max; sleep_cnt++) {\n+\t\tctx.pending_work = true;\n+\n+\t\terr = iocpt_adminq_post_wait(dev, &ctx);\n+\t\tif (err != -EAGAIN)\n+\t\t\tbreak;\n+\n+\t\trte_delay_us_block(sleep_usec);\n+\t}\n+\n+\tif (err != 0)\n+\t\tIOCPT_PRINT(ERR, \"Deinit queue %u returned %d after %u ms\",\n+\t\t\tcptq->q.index, err, sleep_cnt * 100);\n+\telse\n+\t\tIOCPT_PRINT(DEBUG, \"Deinit queue %u returned %d after %u ms\",\n+\t\t\tcptq->q.index, err, sleep_cnt * 100);\n+\n+\tIOCPT_PRINT(DEBUG, \"Queue %u watchdog: enq %\"PRIu64\" deq %\"PRIu64,\n+\t\tcptq->q.index, cptq->enqueued_wdogs, cptq->dequeued_wdogs);\n+\n+\tcptq->flags &= ~IOCPT_Q_F_INITED;\n+}\n+\n+void\n+iocpt_cryptoq_free(struct iocpt_crypto_q *cptq)\n+{\n+\tif (cptq == NULL)\n+\t\treturn;\n+\n+\tif (cptq->base_z != NULL) {\n+\t\trte_memzone_free(cptq->base_z);\n+\t\tcptq->base_z = NULL;\n+\t\tcptq->base = NULL;\n+\t\tcptq->base_pa = 0;\n+\t}\n+\n+\tiocpt_q_free(&cptq->q);\n+\n+\trte_free(cptq);\n+}\n+\n static int\n iocpt_adminq_alloc(struct iocpt_dev *dev)\n {\n@@ -313,6 +646,14 @@ iocpt_alloc_objs(struct iocpt_dev *dev)\n \n \tIOCPT_PRINT(DEBUG, \"Crypto: %s\", dev->name);\n \n+\tdev->cryptoqs = rte_calloc_socket(\"iocpt\",\n+\t\t\t\tdev->max_qps, sizeof(*dev->cryptoqs),\n+\t\t\t\tRTE_CACHE_LINE_SIZE, dev->socket_id);\n+\tif (dev->cryptoqs == NULL) {\n+\t\tIOCPT_PRINT(ERR, \"Cannot allocate tx queues array\");\n+\t\treturn -ENOMEM;\n+\t}\n+\n \trte_spinlock_init(&dev->adminq_lock);\n \trte_spinlock_init(&dev->adminq_service_lock);\n \n@@ -320,7 +661,7 @@ iocpt_alloc_objs(struct iocpt_dev *dev)\n \tif (err != 0) {\n \t\tIOCPT_PRINT(ERR, \"Cannot allocate admin queue\");\n \t\terr = -ENOMEM;\n-\t\tgoto err_out;\n+\t\tgoto err_free_cryptoqs;\n \t}\n \n \tdev->info_sz = RTE_ALIGN(sizeof(*dev->info), rte_mem_page_size());\n@@ -365,7 +706,9 @@ iocpt_alloc_objs(struct iocpt_dev *dev)\n err_free_adminq:\n \tiocpt_adminq_free(dev->adminq);\n \tdev->adminq = NULL;\n-err_out:\n+err_free_cryptoqs:\n+\trte_free(dev->cryptoqs);\n+\tdev->cryptoqs = NULL;\n \treturn err;\n }\n \n@@ -385,9 +728,21 @@ iocpt_init(struct iocpt_dev *dev)\n \tif (err != 0)\n \t\treturn err;\n \n+\t/* Write the queue watchdog key */\n+\terr = iocpt_session_wdog(dev);\n+\tif (err != 0) {\n+\t\tIOCPT_PRINT(ERR, \"Cannot setup watchdog session\");\n+\t\tgoto err_out_adminq_deinit;\n+\t}\n+\n \tdev->state |= IOCPT_DEV_F_INITED;\n \n \treturn 0;\n+\n+err_out_adminq_deinit:\n+\tiocpt_adminq_deinit(dev);\n+\n+\treturn err;\n }\n \n void\n@@ -396,6 +751,43 @@ iocpt_configure(struct iocpt_dev *dev)\n \tRTE_SET_USED(dev);\n }\n \n+int\n+iocpt_start(struct iocpt_dev *dev)\n+{\n+\tuint32_t i;\n+\tint err;\n+\n+\tIOCPT_PRINT(DEBUG, \"Starting %u queues\",\n+\t\tdev->crypto_dev->data->nb_queue_pairs);\n+\n+\tfor (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {\n+\t\terr = iocpt_cryptoq_init(dev->cryptoqs[i]);\n+\t\tif (err != 0)\n+\t\t\treturn err;\n+\t}\n+\n+\tdev->state |= IOCPT_DEV_F_UP;\n+\n+\treturn 0;\n+}\n+\n+void\n+iocpt_stop(struct iocpt_dev *dev)\n+{\n+\tuint32_t i;\n+\n+\tIOCPT_PRINT_CALL();\n+\n+\tdev->state &= ~IOCPT_DEV_F_UP;\n+\n+\tfor (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {\n+\t\tstruct iocpt_crypto_q *cptq = dev->cryptoqs[i];\n+\n+\t\tif (cptq->flags & IOCPT_Q_F_INITED)\n+\t\t\t(void)iocpt_cryptoq_deinit(cptq);\n+\t}\n+}\n+\n void\n iocpt_deinit(struct iocpt_dev *dev)\n {\n@@ -412,8 +804,16 @@ iocpt_deinit(struct iocpt_dev *dev)\n static void\n iocpt_free_objs(struct iocpt_dev *dev)\n {\n+\tvoid **queue_pairs = dev->crypto_dev->data->queue_pairs;\n+\tuint32_t i;\n+\n \tIOCPT_PRINT_CALL();\n \n+\tfor (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {\n+\t\tiocpt_cryptoq_free(queue_pairs[i]);\n+\t\tqueue_pairs[i] = NULL;\n+\t}\n+\n \tif (dev->sess_bm != NULL) {\n \t\trte_bitmap_free(dev->sess_bm);\n \t\trte_free(dev->sess_bm);\n@@ -425,6 +825,11 @@ iocpt_free_objs(struct iocpt_dev *dev)\n \t\tdev->adminq = NULL;\n \t}\n \n+\tif (dev->cryptoqs != NULL) {\n+\t\trte_free(dev->cryptoqs);\n+\t\tdev->cryptoqs = NULL;\n+\t}\n+\n \tif (dev->info != NULL) {\n \t\trte_memzone_free(dev->info_z);\n \t\tdev->info_z = NULL;\n@@ -542,8 +947,16 @@ iocpt_probe(void *bus_dev, struct rte_device *rte_dev,\n \t\tgoto err_free_objs;\n \t}\n \n+\terr = iocpt_assign_ops(cdev);\n+\tif (err != 0) {\n+\t\tIOCPT_PRINT(ERR, \"Failed to configure opts\");\n+\t\tgoto err_deinit_dev;\n+\t}\n+\n \treturn 0;\n \n+err_deinit_dev:\n+\tiocpt_deinit(dev);\n err_free_objs:\n \tiocpt_free_objs(dev);\n err_destroy_crypto_dev:\ndiff --git a/drivers/crypto/ionic/ionic_crypto_ops.c b/drivers/crypto/ionic/ionic_crypto_ops.c\nnew file mode 100644\nindex 0000000000..69768c9955\n--- /dev/null\n+++ b/drivers/crypto/ionic/ionic_crypto_ops.c\n@@ -0,0 +1,606 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2021-2024 Advanced Micro Devices, Inc.\n+ */\n+\n+#include <rte_cryptodev.h>\n+#include <cryptodev_pmd.h>\n+#include <rte_errno.h>\n+#include <rte_malloc.h>\n+#include <rte_mempool.h>\n+\n+#include \"ionic_crypto.h\"\n+\n+static int\n+iocpt_op_config(struct rte_cryptodev *cdev,\n+\t\tstruct rte_cryptodev_config *config __rte_unused)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\tiocpt_configure(dev);\n+\n+\treturn 0;\n+}\n+\n+static int\n+iocpt_op_start(struct rte_cryptodev *cdev)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\treturn iocpt_start(dev);\n+}\n+\n+static void\n+iocpt_op_stop(struct rte_cryptodev *cdev)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\treturn iocpt_stop(dev);\n+}\n+\n+static int\n+iocpt_op_close(struct rte_cryptodev *cdev)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\tiocpt_deinit(dev);\n+\n+\treturn 0;\n+}\n+\n+static void\n+iocpt_op_info_get(struct rte_cryptodev *cdev, struct rte_cryptodev_info *info)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\tif (info == NULL)\n+\t\treturn;\n+\n+\tinfo->max_nb_queue_pairs = dev->max_qps;\n+\tinfo->feature_flags = dev->features;\n+\tinfo->capabilities = iocpt_get_caps(info->feature_flags);\n+\t/* Reserve one session for watchdog */\n+\tinfo->sym.max_nb_sessions = dev->max_sessions - 1;\n+\tinfo->driver_id = dev->driver_id;\n+\tinfo->min_mbuf_headroom_req = 0;\n+\tinfo->min_mbuf_tailroom_req = 0;\n+}\n+\n+static void\n+iocpt_op_stats_get(struct rte_cryptodev *cdev,\n+\t\tstruct rte_cryptodev_stats *stats)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\tiocpt_get_stats(dev, stats);\n+}\n+\n+static void\n+iocpt_op_stats_reset(struct rte_cryptodev *cdev)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\tiocpt_reset_stats(dev);\n+}\n+\n+static int\n+iocpt_op_queue_release(struct rte_cryptodev *cdev, uint16_t queue_id)\n+{\n+\tstruct iocpt_crypto_q *cptq = cdev->data->queue_pairs[queue_id];\n+\n+\tIOCPT_PRINT(DEBUG, \"queue_id %u\", queue_id);\n+\n+\tassert(!(cptq->flags & IOCPT_Q_F_INITED));\n+\n+\tiocpt_cryptoq_free(cptq);\n+\n+\tcdev->data->queue_pairs[queue_id] = NULL;\n+\n+\treturn 0;\n+}\n+\n+static int\n+iocpt_op_queue_setup(struct rte_cryptodev *cdev, uint16_t queue_id,\n+\t\tconst struct rte_cryptodev_qp_conf *qp_conf,\n+\t\tint socket_id)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\tint err;\n+\n+\tif (cdev->data->queue_pairs[queue_id] != NULL)\n+\t\tiocpt_op_queue_release(cdev, queue_id);\n+\n+\tif (qp_conf->nb_descriptors < (1 << IOCPT_QSIZE_MIN_LG2) ||\n+\t qp_conf->nb_descriptors > (1 << IOCPT_QSIZE_MAX_LG2)) {\n+\t\tIOCPT_PRINT(ERR, \"invalid nb_descriptors %u, use range %u..%u\",\n+\t\t\tqp_conf->nb_descriptors,\n+\t\t\t1 << IOCPT_QSIZE_MIN_LG2, 1 << IOCPT_QSIZE_MAX_LG2);\n+\t\treturn -ERANGE;\n+\t}\n+\n+\tIOCPT_PRINT(DEBUG, \"queue_id %u\", queue_id);\n+\n+\terr = iocpt_cryptoq_alloc(dev, socket_id, queue_id,\n+\t\t\t\tqp_conf->nb_descriptors);\n+\tif (err != 0)\n+\t\treturn err;\n+\n+\tcdev->data->queue_pairs[queue_id] = dev->cryptoqs[queue_id];\n+\n+\treturn 0;\n+}\n+\n+static unsigned int\n+iocpt_op_get_session_size(struct rte_cryptodev *cdev __rte_unused)\n+{\n+\treturn iocpt_session_size();\n+}\n+\n+static inline int\n+iocpt_is_algo_supported(struct rte_crypto_sym_xform *xform)\n+{\n+\tif (xform->next != NULL) {\n+\t\tIOCPT_PRINT(ERR, \"chaining not supported\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\tif (xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) {\n+\t\tIOCPT_PRINT(ERR, \"xform->type %d not supported\", xform->type);\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline int\n+iocpt_fill_sess_aead(struct rte_crypto_sym_xform *xform,\n+\t\tstruct iocpt_session_priv *priv)\n+{\n+\tstruct rte_crypto_aead_xform *aead_form = &xform->aead;\n+\n+\tif (aead_form->algo != RTE_CRYPTO_AEAD_AES_GCM) {\n+\t\tIOCPT_PRINT(ERR, \"Unknown algo\");\n+\t\treturn -EINVAL;\n+\t}\n+\tif (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {\n+\t\tpriv->op = IOCPT_DESC_OPCODE_GCM_AEAD_ENCRYPT;\n+\t} else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) {\n+\t\tpriv->op = IOCPT_DESC_OPCODE_GCM_AEAD_DECRYPT;\n+\t} else {\n+\t\tIOCPT_PRINT(ERR, \"Unknown cipher operations\");\n+\t\treturn -1;\n+\t}\n+\n+\tif (aead_form->key.length < IOCPT_SESS_KEY_LEN_MIN ||\n+\t aead_form->key.length > IOCPT_SESS_KEY_LEN_MAX_SYMM) {\n+\t\tIOCPT_PRINT(ERR, \"Invalid cipher keylen %u\",\n+\t\t\taead_form->key.length);\n+\t\treturn -1;\n+\t}\n+\tpriv->key_len = aead_form->key.length;\n+\tmemcpy(priv->key, aead_form->key.data, priv->key_len);\n+\n+\tpriv->type = IOCPT_SESS_AEAD_AES_GCM;\n+\tpriv->iv_offset = aead_form->iv.offset;\n+\tpriv->iv_length = aead_form->iv.length;\n+\tpriv->digest_length = aead_form->digest_length;\n+\tpriv->aad_length = aead_form->aad_length;\n+\n+\treturn 0;\n+}\n+\n+static int\n+iocpt_session_cfg(struct iocpt_dev *dev,\n+\t\t struct rte_crypto_sym_xform *xform,\n+\t\t struct rte_cryptodev_sym_session *sess)\n+{\n+\tstruct rte_crypto_sym_xform *chain;\n+\tstruct iocpt_session_priv *priv = NULL;\n+\n+\tif (iocpt_is_algo_supported(xform) < 0)\n+\t\treturn -ENOTSUP;\n+\n+\tif (unlikely(sess == NULL)) {\n+\t\tIOCPT_PRINT(ERR, \"invalid session\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tpriv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);\n+\tpriv->dev = dev;\n+\n+\tchain = xform;\n+\twhile (chain) {\n+\t\tswitch (chain->type) {\n+\t\tcase RTE_CRYPTO_SYM_XFORM_AEAD:\n+\t\t\tif (iocpt_fill_sess_aead(chain, priv))\n+\t\t\t\treturn -EIO;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tIOCPT_PRINT(ERR, \"invalid crypto xform type %d\",\n+\t\t\t\tchain->type);\n+\t\t\treturn -ENOTSUP;\n+\t\t}\n+\t\tchain = chain->next;\n+\t}\n+\n+\treturn iocpt_session_init(priv);\n+}\n+\n+static int\n+iocpt_op_session_cfg(struct rte_cryptodev *cdev,\n+\t\tstruct rte_crypto_sym_xform *xform,\n+\t\tstruct rte_cryptodev_sym_session *sess)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\treturn iocpt_session_cfg(dev, xform, sess);\n+}\n+\n+static void\n+iocpt_session_clear(struct rte_cryptodev_sym_session *sess)\n+{\n+\tiocpt_session_deinit(CRYPTODEV_GET_SYM_SESS_PRIV(sess));\n+}\n+\n+static void\n+iocpt_op_session_clear(struct rte_cryptodev *cdev __rte_unused,\n+\t\tstruct rte_cryptodev_sym_session *sess)\n+{\n+\tiocpt_session_clear(sess);\n+}\n+\n+static inline void\n+iocpt_fill_sge(struct iocpt_crypto_sg_elem *arr, uint8_t idx,\n+\t uint64_t addr, uint16_t len)\n+{\n+\tarr[idx].addr = rte_cpu_to_le_64(addr);\n+\tarr[idx].len = rte_cpu_to_le_16(len);\n+}\n+\n+static __rte_always_inline int\n+iocpt_enq_one_aead(struct iocpt_crypto_q *cptq,\n+\t\t struct iocpt_session_priv *priv, struct rte_crypto_op *op)\n+{\n+\tstruct rte_crypto_sym_op *sym_op = op->sym;\n+\tstruct iocpt_queue *q = &cptq->q;\n+\tstruct iocpt_crypto_desc *desc, *desc_base = q->base;\n+\tstruct iocpt_crypto_sg_desc *sg_desc, *sg_desc_base = q->sg_base;\n+\tstruct iocpt_crypto_sg_elem *src, *dst;\n+\trte_iova_t aad_addr, digest_addr, iv_addr, seg_addr;\n+\tuint32_t data_len, data_offset, seg_len;\n+\tuint8_t nsge_src = 0, nsge_dst = 0, flags = 0;\n+\tstruct rte_mbuf *m;\n+\n+\tdesc = &desc_base[q->head_idx];\n+\tsg_desc = &sg_desc_base[q->head_idx];\n+\tsrc = sg_desc->src_elems;\n+\tdst = sg_desc->dst_elems;\n+\n+\t/* Fill the first SGE with the IV / Nonce */\n+\tiv_addr = rte_crypto_op_ctophys_offset(op, priv->iv_offset);\n+\tiocpt_fill_sge(src, nsge_src++, iv_addr, priv->iv_length);\n+\n+\t/* Fill the second SGE with the AAD, if applicable */\n+\tif (priv->aad_length > 0) {\n+\t\taad_addr = sym_op->aead.aad.phys_addr;\n+\t\tiocpt_fill_sge(src, nsge_src++, aad_addr, priv->aad_length);\n+\t\tflags |= IOCPT_DESC_F_AAD_VALID;\n+\t}\n+\n+\tm = sym_op->m_src;\n+\tdata_len = sym_op->aead.data.length;\n+\n+\t/* Fast-forward through mbuf chain to account for data offset */\n+\tdata_offset = sym_op->aead.data.offset;\n+\twhile (m != NULL && data_offset >= m->data_len) {\n+\t\tdata_offset -= m->data_len;\n+\t\tm = m->next;\n+\t}\n+\n+\t/* Fill the next SGEs with the payload segments */\n+\twhile (m != NULL && data_len > 0) {\n+\t\tseg_addr = rte_mbuf_data_iova(m) + data_offset;\n+\t\tseg_len = RTE_MIN(m->data_len - data_offset, data_len);\n+\t\tdata_offset = 0;\n+\t\tdata_len -= seg_len;\n+\n+\t\t/* Use -1 to save room for digest */\n+\t\tif (nsge_src >= IOCPT_CRYPTO_MAX_SG_ELEMS - 1)\n+\t\t\treturn -ERANGE;\n+\n+\t\tiocpt_fill_sge(src, nsge_src++, seg_addr, seg_len);\n+\n+\t\tm = m->next;\n+\t}\n+\n+\t/* AEAD AES-GCM: digest == authentication tag */\n+\tdigest_addr = sym_op->aead.digest.phys_addr;\n+\tiocpt_fill_sge(src, nsge_src++, digest_addr, priv->digest_length);\n+\n+\t/* Process Out-Of-Place destination SGL */\n+\tif (sym_op->m_dst != NULL) {\n+\t\t/* Put the AAD here, too */\n+\t\tif (priv->aad_length > 0)\n+\t\t\tiocpt_fill_sge(dst, nsge_dst++,\n+\t\t\t\tsym_op->aead.aad.phys_addr, priv->aad_length);\n+\n+\t\tm = sym_op->m_dst;\n+\t\tdata_len = sym_op->aead.data.length;\n+\n+\t\t/* Fast-forward through chain to account for data offset */\n+\t\tdata_offset = sym_op->aead.data.offset;\n+\t\twhile (m != NULL && data_offset >= m->data_len) {\n+\t\t\tdata_offset -= m->data_len;\n+\t\t\tm = m->next;\n+\t\t}\n+\n+\t\t/* Fill in the SGEs with the payload segments */\n+\t\twhile (m != NULL && data_len > 0) {\n+\t\t\tseg_addr = rte_mbuf_data_iova(m) + data_offset;\n+\t\t\tseg_len = RTE_MIN(m->data_len - data_offset, data_len);\n+\t\t\tdata_offset = 0;\n+\t\t\tdata_len -= seg_len;\n+\n+\t\t\tif (nsge_dst >= IOCPT_CRYPTO_MAX_SG_ELEMS)\n+\t\t\t\treturn -ERANGE;\n+\n+\t\t\tiocpt_fill_sge(dst, nsge_dst++, seg_addr, seg_len);\n+\n+\t\t\tm = m->next;\n+\t\t}\n+\t}\n+\n+\tdesc->opcode = priv->op;\n+\tdesc->flags = flags;\n+\tdesc->num_src_dst_sgs = iocpt_encode_nsge_src_dst(nsge_src, nsge_dst);\n+\tdesc->session_tag = rte_cpu_to_le_32(priv->index);\n+\n+\top->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;\n+\tq->info[q->head_idx] = op;\n+\tq->head_idx = Q_NEXT_TO_POST(q, 1);\n+\n+\treturn 0;\n+}\n+\n+static uint16_t\n+iocpt_enqueue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n+{\n+\tstruct iocpt_crypto_q *cptq = qp;\n+\tstruct rte_crypto_op *op;\n+\tstruct iocpt_session_priv *priv;\n+\tstruct rte_cryptodev_stats *stats = &cptq->stats;\n+\tuint16_t avail, count;\n+\tint err;\n+\n+\tavail = iocpt_q_space_avail(&cptq->q);\n+\tif (unlikely(nb_ops > avail))\n+\t\tnb_ops = avail;\n+\n+\tcount = 0;\n+\twhile (likely(count < nb_ops)) {\n+\t\top = ops[count];\n+\n+\t\tif (unlikely(op->sess_type != RTE_CRYPTO_OP_WITH_SESSION)) {\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tpriv = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session);\n+\t\tif (unlikely(priv == NULL)) {\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\terr = iocpt_enq_one_aead(cptq, priv, op);\n+\t\tif (unlikely(err != 0)) {\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;\n+\t\t\tstats->enqueue_err_count++;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tcount++;\n+\t}\n+\n+\tif (likely(count > 0)) {\n+\t\tiocpt_q_flush(&cptq->q);\n+\n+\t\t/* Restart timer if ops are being enqueued */\n+\t\tcptq->last_wdog_cycles = rte_get_timer_cycles();\n+\n+\t\tstats->enqueued_count += count;\n+\t}\n+\n+\treturn count;\n+}\n+\n+static void\n+iocpt_enqueue_wdog(struct iocpt_crypto_q *cptq)\n+{\n+\tstruct iocpt_queue *q = &cptq->q;\n+\tstruct iocpt_crypto_desc *desc, *desc_base = q->base;\n+\tstruct iocpt_crypto_sg_desc *sg_desc, *sg_desc_base = q->sg_base;\n+\tstruct iocpt_crypto_sg_elem *src;\n+\tstruct rte_crypto_op *wdog_op;\n+\trte_iova_t iv_addr, pld_addr, tag_addr;\n+\tuint8_t nsge_src = 0;\n+\tuint16_t avail;\n+\n+\tavail = iocpt_q_space_avail(&cptq->q);\n+\tif (avail < 1)\n+\t\tgoto out_flush;\n+\n+\twdog_op = rte_zmalloc_socket(\"iocpt\", sizeof(*wdog_op),\n+\t\t\t\tRTE_CACHE_LINE_SIZE, rte_socket_id());\n+\tif (wdog_op == NULL)\n+\t\tgoto out_flush;\n+\n+\twdog_op->type = IOCPT_Q_WDOG_OP_TYPE;\n+\twdog_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;\n+\n+\tdesc = &desc_base[q->head_idx];\n+\tsg_desc = &sg_desc_base[q->head_idx];\n+\tsrc = sg_desc->src_elems;\n+\n+\t/* Fill the first SGE with the IV / Nonce */\n+\tiv_addr = rte_mem_virt2iova(cptq->wdog_iv);\n+\tiocpt_fill_sge(src, nsge_src++, iv_addr, IOCPT_Q_WDOG_IV_LEN);\n+\n+\t/* Fill the second SGE with the payload segment */\n+\tpld_addr = rte_mem_virt2iova(cptq->wdog_pld);\n+\tiocpt_fill_sge(src, nsge_src++, pld_addr, IOCPT_Q_WDOG_PLD_LEN);\n+\n+\t/* AEAD AES-GCM: digest == authentication tag */\n+\ttag_addr = rte_mem_virt2iova(cptq->wdog_tag);\n+\tiocpt_fill_sge(src, nsge_src++, tag_addr, IOCPT_Q_WDOG_TAG_LEN);\n+\n+\tdesc->opcode = IOCPT_DESC_OPCODE_GCM_AEAD_ENCRYPT;\n+\tdesc->flags = 0;\n+\tdesc->num_src_dst_sgs = iocpt_encode_nsge_src_dst(nsge_src, 0);\n+\tdesc->session_tag = rte_cpu_to_le_32(IOCPT_Q_WDOG_SESS_IDX);\n+\n+\tq->info[q->head_idx] = wdog_op;\n+\tq->head_idx = Q_NEXT_TO_POST(q, 1);\n+\n+\tIOCPT_PRINT(DEBUG, \"Queue %u wdog enq %p ops %\"PRIu64,\n+\t\tq->index, wdog_op, cptq->stats.enqueued_count);\n+\tcptq->enqueued_wdogs++;\n+\n+out_flush:\n+\tiocpt_q_flush(q);\n+}\n+\n+static uint16_t\n+iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)\n+{\n+\tstruct iocpt_crypto_q *cptq = qp;\n+\tstruct iocpt_queue *q = &cptq->q;\n+\tstruct iocpt_cq *cq = &cptq->cq;\n+\tstruct rte_crypto_op *op;\n+\tstruct iocpt_crypto_comp *cq_desc_base = cq->base;\n+\tvolatile struct iocpt_crypto_comp *cq_desc;\n+\tstruct rte_cryptodev_stats *stats = &cptq->stats;\n+\tuint64_t then, now, hz, delta;\n+\tuint16_t count = 0;\n+\n+\tcq_desc = &cq_desc_base[cq->tail_idx];\n+\n+\t/* First walk the CQ to update any completed op's status\n+\t * NB: These can arrive out of order!\n+\t */\n+\twhile ((cq_desc->color & 0x1) == cq->done_color) {\n+\t\tcq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);\n+\t\tif (unlikely(cq->tail_idx == 0))\n+\t\t\tcq->done_color = !cq->done_color;\n+\n+\t\top = q->info[rte_le_to_cpu_16(cq_desc->comp_index)];\n+\n+\t\t/* Process returned CQ descriptor status */\n+\t\tif (unlikely(cq_desc->status)) {\n+\t\t\tswitch (cq_desc->status) {\n+\t\t\tcase IOCPT_COMP_SYMM_AUTH_VERIFY_ERROR:\n+\t\t\t\top->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n+\t\t\t\tbreak;\n+\t\t\tcase IOCPT_COMP_INVAL_OPCODE_ERROR:\n+\t\t\tcase IOCPT_COMP_UNSUPP_OPCODE_ERROR:\n+\t\t\tcase IOCPT_COMP_SYMM_SRC_SG_ERROR:\n+\t\t\tcase IOCPT_COMP_SYMM_DST_SG_ERROR:\n+\t\t\tcase IOCPT_COMP_SYMM_SRC_DST_LEN_MISMATCH:\n+\t\t\tcase IOCPT_COMP_SYMM_KEY_IDX_ERROR:\n+\t\t\t\top->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;\n+\t\t\t\tbreak;\n+\t\t\tdefault:\n+\t\t\t\top->status = RTE_CRYPTO_OP_STATUS_ERROR;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t} else\n+\t\t\top->status = RTE_CRYPTO_OP_STATUS_SUCCESS;\n+\n+\t\tcq_desc = &cq_desc_base[cq->tail_idx];\n+\t}\n+\n+\t/* Next walk the SQ to pop off completed ops in-order */\n+\twhile (count < nb_ops) {\n+\t\top = q->info[q->tail_idx];\n+\n+\t\t/* No more completions */\n+\t\tif (op == NULL ||\n+\t\t op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)\n+\t\t\tbreak;\n+\n+\t\t/* Handle watchdog operations */\n+\t\tif (unlikely(op->type == IOCPT_Q_WDOG_OP_TYPE)) {\n+\t\t\tIOCPT_PRINT(DEBUG, \"Queue %u wdog deq %p st %d\",\n+\t\t\t\tq->index, op, op->status);\n+\t\t\tq->info[q->tail_idx] = NULL;\n+\t\t\tq->tail_idx = Q_NEXT_TO_SRVC(q, 1);\n+\t\t\tcptq->dequeued_wdogs++;\n+\t\t\trte_free(op);\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tif (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))\n+\t\t\tstats->dequeue_err_count++;\n+\n+\t\tops[count] = op;\n+\t\tq->info[q->tail_idx] = NULL;\n+\n+\t\tq->tail_idx = Q_NEXT_TO_SRVC(q, 1);\n+\t\tcount++;\n+\t}\n+\n+\tif (!count) {\n+\t\t/*\n+\t\t * Ring the doorbell again if no work was dequeued and work\n+\t\t * is still pending after the deadline.\n+\t\t */\n+\t\tif (q->head_idx != q->tail_idx) {\n+\t\t\tthen = cptq->last_wdog_cycles;\n+\t\t\tnow = rte_get_timer_cycles();\n+\t\t\thz = rte_get_timer_hz();\n+\t\t\tdelta = (now - then) * 1000;\n+\n+\t\t\tif (delta >= hz * IONIC_Q_WDOG_MS) {\n+\t\t\t\tiocpt_enqueue_wdog(cptq);\n+\t\t\t\tcptq->last_wdog_cycles = now;\n+\t\t\t}\n+\t\t}\n+\t} else\n+\t\t/* Restart timer if the queue is making progress */\n+\t\tcptq->last_wdog_cycles = rte_get_timer_cycles();\n+\n+\tstats->dequeued_count += count;\n+\n+\treturn count;\n+}\n+\n+static struct rte_cryptodev_ops iocpt_ops = {\n+\t.dev_configure = iocpt_op_config,\n+\t.dev_start = iocpt_op_start,\n+\t.dev_stop = iocpt_op_stop,\n+\t.dev_close = iocpt_op_close,\n+\t.dev_infos_get = iocpt_op_info_get,\n+\n+\t.stats_get = iocpt_op_stats_get,\n+\t.stats_reset = iocpt_op_stats_reset,\n+\t.queue_pair_setup = iocpt_op_queue_setup,\n+\t.queue_pair_release = iocpt_op_queue_release,\n+\n+\t.sym_session_get_size = iocpt_op_get_session_size,\n+\t.sym_session_configure = iocpt_op_session_cfg,\n+\t.sym_session_clear = iocpt_op_session_clear,\n+};\n+\n+int\n+iocpt_assign_ops(struct rte_cryptodev *cdev)\n+{\n+\tstruct iocpt_dev *dev = cdev->data->dev_private;\n+\n+\tcdev->dev_ops = &iocpt_ops;\n+\tcdev->feature_flags = dev->features;\n+\n+\tif (dev->features & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) {\n+\t\tcdev->enqueue_burst = iocpt_enqueue_sym;\n+\t\tcdev->dequeue_burst = iocpt_dequeue_sym;\n+\t}\n+\n+\treturn 0;\n+}\ndiff --git a/drivers/crypto/ionic/meson.build b/drivers/crypto/ionic/meson.build\nindex a6e0a1d415..b63428fa9b 100644\n--- a/drivers/crypto/ionic/meson.build\n+++ b/drivers/crypto/ionic/meson.build\n@@ -5,8 +5,10 @@ deps += ['bus_vdev']\n deps += ['common_ionic']\n \n sources = files(\n+ 'ionic_crypto_caps.c',\n 'ionic_crypto_cmds.c',\n 'ionic_crypto_main.c',\n+ 'ionic_crypto_ops.c',\n 'ionic_crypto_vdev.c',\n )\n name = 'ionic_crypto'\n", "prefixes": [ "5/6" ] }