Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/118816/?format=api
{ "id": 118816, "url": "http://patches.dpdk.org/api/patches/118816/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20221020155749.16643-11-valex@nvidia.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20221020155749.16643-11-valex@nvidia.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20221020155749.16643-11-valex@nvidia.com", "date": "2022-10-20T15:57:40", "name": "[v6,10/18] net/mlx5/hws: Add HWS send layer", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "8f80957bf6fe9d666ee995a1b87d77b40cd5f87c", "submitter": { "id": 2858, "url": "http://patches.dpdk.org/api/people/2858/?format=api", "name": "Alex Vesker", "email": "valex@nvidia.com" }, "delegate": { "id": 3268, "url": "http://patches.dpdk.org/api/users/3268/?format=api", "username": "rasland", "first_name": "Raslan", "last_name": "Darawsheh", "email": "rasland@nvidia.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20221020155749.16643-11-valex@nvidia.com/mbox/", "series": [ { "id": 25345, "url": "http://patches.dpdk.org/api/series/25345/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=25345", "date": "2022-10-20T15:57:30", "name": "net/mlx5: Add HW steering low level support", "version": 6, "mbox": "http://patches.dpdk.org/series/25345/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/118816/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/118816/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 7BE83A0554;\n\tThu, 20 Oct 2022 18:00:03 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 5EBD742BB9;\n\tThu, 20 Oct 2022 17:59:25 +0200 (CEST)", "from NAM10-MW2-obe.outbound.protection.outlook.com\n (mail-mw2nam10on2057.outbound.protection.outlook.com [40.107.94.57])\n by mails.dpdk.org (Postfix) with ESMTP id 0E63342B78\n for <dev@dpdk.org>; Thu, 20 Oct 2022 17:59:22 +0200 (CEST)", "from DS7PR03CA0340.namprd03.prod.outlook.com (2603:10b6:8:55::21) by\n DM6PR12MB4139.namprd12.prod.outlook.com (2603:10b6:5:214::18) with\n Microsoft\n SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id\n 15.20.5723.35; Thu, 20 Oct 2022 15:59:19 +0000", "from DM6NAM11FT099.eop-nam11.prod.protection.outlook.com\n (2603:10b6:8:55:cafe::72) by DS7PR03CA0340.outlook.office365.com\n (2603:10b6:8:55::21) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5723.35 via Frontend\n Transport; Thu, 20 Oct 2022 15:59:19 +0000", "from mail.nvidia.com (216.228.117.160) by\n DM6NAM11FT099.mail.protection.outlook.com (10.13.172.241) with Microsoft SMTP\n Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id\n 15.20.5746.16 via Frontend Transport; Thu, 20 Oct 2022 15:59:19 +0000", "from rnnvmail201.nvidia.com (10.129.68.8) by mail.nvidia.com\n (10.129.200.66) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.986.26; Thu, 20 Oct\n 2022 08:59:12 -0700", "from nvidia.com (10.126.230.35) by rnnvmail201.nvidia.com\n (10.129.68.8) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.986.29; Thu, 20 Oct\n 2022 08:59:09 -0700" ], "ARC-Seal": "i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none;\n b=WAScTYuwBAmt8KrWyn/+SMNhTW8O8u0Jzex6v6m0HDGdduLXdDDkWVJT363kJPJ4giZ0deCeKpk/YAphd9Jxd01wrsjuK86mb3kakH1A0seylUGlpPI4AwI05KTPtNB6xk352eY6ZDCY0SyCULK8c3LULGCoavBmIE5Tdrvi8QMXMc6r0RDSIGOVNtOepHgqZP9tC8q3qsMUvHIjWmy0NIn+8uS5qhCQ0a07cWk3F9tpMc3nNKCHnYixLjBzPCU+SxFTyTYN6EsmaQax9+p9q4BqvWHXhnAqrYOcqj+emPrMUWkFXZ8wQr26YL7BRSjJTvieILp1+IiXXb8Oj+R1SA==", "ARC-Message-Signature": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com;\n s=arcselector9901;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1;\n bh=ZhQKJLvgtf8XGMhek3gqJ8eHoEXSmXXgpkcMsfcT29c=;\n b=jOzHc5LJOBViH8RW6NmwsrcpuPO5AxpGnqqoi9A70ovmLnM8BV0p2pfBEc71fra1kT3ifLEFb5Q1jrsdXRkHgW0/ZgM1D20jXMzIRv/5SDEbOIj4ChQmHEkXZFXcWoWNZfzO0kB/0FpH2O3qkf5ZfOSrtSE1JeP7l8zKFtBMIJ5sprEL78yeXidK1Gb0vGnQ29oLkd/3X3DIoDQNyx1xgd3Io/8z5XJZDWxHEzadVXchOZpOYNYYJZxjspWSpjOVeeBRR/kkO20H+AFYlnsSSU8ynVdN+hdgtMQR0N+16iHO6Tci5qHvJJeRwEtJUDniNrI/9CxoqfAkGCoIBt+jjw==", "ARC-Authentication-Results": "i=1; mx.microsoft.com 1; spf=none (sender ip is\n 216.228.117.160) smtp.rcpttodomain=monjalon.net smtp.mailfrom=nvidia.com;\n dmarc=fail (p=reject sp=reject pct=100) action=oreject\n header.from=nvidia.com; dkim=none (message not signed); arc=none", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=Nvidia.com;\n s=selector2;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=ZhQKJLvgtf8XGMhek3gqJ8eHoEXSmXXgpkcMsfcT29c=;\n b=k+EiFHhK7z7bypExf/vDwlyEvTCoBaaXsg10QyH09J4Fw+TxHGVvLldU9dBI3re8XdvEkd2Gu8CGHN2YjMM1F3EpS3RBL7aJTM+cHLFNioSShWopsH0m6iuskX113SzphVXBkNxcrbO04fn/U5ZwaqREBph2eLgbLch7uX6zmNiY6734lCp510u3zketHcmSVE7vxWV8K7QyxxRj+NoedbDfe/E2W6FNygRD7BvmZWHmxYntJZPnmXKsYW8Ywds3W0q5y5lQOBnjfiluew/q9qiVhK/sjNrpuosP2VeR3dGbh7lmt4NI9NRR/X8ts5GlPiVDJA4LEPDCUtifyjnJdQ==", "X-MS-Exchange-Authentication-Results": "spf=none (sender IP is 216.228.117.160)\n smtp.mailfrom=nvidia.com;\n dkim=none (message not signed)\n header.d=none;dmarc=fail action=oreject header.from=nvidia.com;", "Received-SPF": "None (protection.outlook.com: nvidia.com does not designate\n permitted sender hosts)", "From": "Alex Vesker <valex@nvidia.com>", "To": "<valex@nvidia.com>, <viacheslavo@nvidia.com>, <thomas@monjalon.net>,\n <suanmingm@nvidia.com>, Matan Azrad <matan@nvidia.com>", "CC": "<dev@dpdk.org>, <orika@nvidia.com>, Mark Bloch <mbloch@nvidia.com>", "Subject": "[v6 10/18] net/mlx5/hws: Add HWS send layer", "Date": "Thu, 20 Oct 2022 18:57:40 +0300", "Message-ID": "<20221020155749.16643-11-valex@nvidia.com>", "X-Mailer": "git-send-email 2.18.1", "In-Reply-To": "<20221020155749.16643-1-valex@nvidia.com>", "References": "<20220922190345.394-1-valex@nvidia.com>\n <20221020155749.16643-1-valex@nvidia.com>", "MIME-Version": "1.0", "Content-Type": "text/plain", "X-Originating-IP": "[10.126.230.35]", "X-ClientProxiedBy": "rnnvmail202.nvidia.com (10.129.68.7) To\n rnnvmail201.nvidia.com (10.129.68.8)", "X-EOPAttributedMessage": "0", "X-MS-PublicTrafficType": "Email", "X-MS-TrafficTypeDiagnostic": "DM6NAM11FT099:EE_|DM6PR12MB4139:EE_", "X-MS-Office365-Filtering-Correlation-Id": "974b1acb-d3b0-4d16-4e21-08dab2b40c43", "X-LD-Processed": "43083d15-7273-40c1-b7db-39efd9ccc17a,ExtAddr", "X-MS-Exchange-SenderADCheck": "1", "X-MS-Exchange-AntiSpam-Relay": "0", "X-Microsoft-Antispam": "BCL:0;", "X-Microsoft-Antispam-Message-Info": "\n 0MIhxMqwA+pdaTAaanJZoZh0YUIgxue88X2j030c6z1XGzJW/isvRFsn6Yr0C0e/z1ZrH9tDpJ+Q7NhMnN2UDn+yx//oUWGwtZtJMhFAb7gJjJzYXUC4blm1JmySJo6JPyiZoIi06kIiH26pVJvBKOiigi/uLvRXf1ofCzZ89O1P0Y9YJCxDDrX0FoDJNel5IMpboWrV7DSkikACp1gqTRBaP3d5Of2Az+MyhU2es56Es9hISDLp6+oq2e7f+WaKpDgNFoUl8V35/ZzuJH1U67dtZvvQfVKhDby5jVN7K+EMrXHBe5AWFXBvODujXTXbmg3sn77yUSZqyy91Ce5bRPjvLsXmTwUu1EmcR3Ss5nV+3aUWjVzCo7hqop22oVh8g+n7QNTTVnV6Oxmh2KLAmHcKWWkPVv5ggtCfCtn1XsbRjagkpr7Qk/mqeioposga+Anmix/K2aUSUzcb9FGp9FrjTunIU8Q1cYwIT4L+7BW7lD8kpPRzrmrLNNxgyaq+dKDEJWSdsImTT/E5qwBaS5xbK0cIeGrg/e3agDaG0Fh5QIf6tS97o0Hp9B4zmjv01K8EM3oe+4XFli8DzouNZjfbZGeZIhel4kDuOiOiNuDdnnHgGQouWp5iK7acuQcjzdwX0PH0M1omrDrHC/X20CD1UKn+dMWC5dC3gLkVXqoF9cM6gFCzu2fpA1KvnOX81pCwzl+p9o4IJY7cd0PMNBo0kMcXt+uWluGxDZ+tfboezFR9Mqho4teyEaAtTexZ01hMeduKiLWv+VZrSXUmag==", "X-Forefront-Antispam-Report": "CIP:216.228.117.160; CTRY:US; LANG:en; SCL:1;\n SRV:;\n IPV:NLI; SFV:NSPM; H:mail.nvidia.com; PTR:dc6edge1.nvidia.com; CAT:NONE;\n SFS:(13230022)(4636009)(396003)(136003)(346002)(39860400002)(376002)(451199015)(36840700001)(46966006)(40470700004)(110136005)(5660300002)(316002)(54906003)(6636002)(4326008)(336012)(8676002)(70206006)(70586007)(41300700001)(6286002)(26005)(478600001)(2906002)(8936002)(30864003)(7696005)(2616005)(16526019)(1076003)(186003)(83380400001)(82740400003)(356005)(7636003)(82310400005)(426003)(47076005)(36860700001)(6666004)(107886003)(36756003)(40480700001)(55016003)(86362001)(40460700003);\n DIR:OUT; SFP:1101;", "X-OriginatorOrg": "Nvidia.com", "X-MS-Exchange-CrossTenant-OriginalArrivalTime": "20 Oct 2022 15:59:19.4379 (UTC)", "X-MS-Exchange-CrossTenant-Network-Message-Id": "\n 974b1acb-d3b0-4d16-4e21-08dab2b40c43", "X-MS-Exchange-CrossTenant-Id": "43083d15-7273-40c1-b7db-39efd9ccc17a", "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp": "\n TenantId=43083d15-7273-40c1-b7db-39efd9ccc17a; Ip=[216.228.117.160];\n Helo=[mail.nvidia.com]", "X-MS-Exchange-CrossTenant-AuthSource": "\n DM6NAM11FT099.eop-nam11.prod.protection.outlook.com", "X-MS-Exchange-CrossTenant-AuthAs": "Anonymous", "X-MS-Exchange-CrossTenant-FromEntityHeader": "HybridOnPrem", "X-MS-Exchange-Transport-CrossTenantHeadersStamped": "DM6PR12MB4139", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "HWS configures flows to the HW using a QP, each WQE has\nthe details of the flow we want to offload. The send layer\nallocates the resources needed to send the request to the HW\nas well as managing the queues, getting completions and\nhandling failures.\n\nSigned-off-by: Mark Bloch <mbloch@nvidia.com>\nSigned-off-by: Alex Vesker <valex@nvidia.com>\n---\n drivers/net/mlx5/hws/mlx5dr_send.c | 844 +++++++++++++++++++++++++++++\n drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++++++++++\n 2 files changed, 1119 insertions(+)\n create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.c\n create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.h", "diff": "diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c b/drivers/net/mlx5/hws/mlx5dr_send.c\nnew file mode 100644\nindex 0000000000..26904a9040\n--- /dev/null\n+++ b/drivers/net/mlx5/hws/mlx5dr_send.c\n@@ -0,0 +1,844 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates\n+ */\n+\n+#include \"mlx5dr_internal.h\"\n+\n+struct mlx5dr_send_ring_dep_wqe *\n+mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue)\n+{\n+\tstruct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;\n+\tunsigned int idx = send_sq->head_dep_idx++ & (queue->num_entries - 1);\n+\n+\tmemset(&send_sq->dep_wqe[idx].wqe_data.tag, 0, MLX5DR_MATCH_TAG_SZ);\n+\n+\treturn &send_sq->dep_wqe[idx];\n+}\n+\n+void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue)\n+{\n+\tqueue->send_ring->send_sq.head_dep_idx--;\n+}\n+\n+void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue)\n+{\n+\tstruct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;\n+\tstruct mlx5dr_send_ste_attr ste_attr = {0};\n+\tstruct mlx5dr_send_ring_dep_wqe *dep_wqe;\n+\n+\tste_attr.send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE;\n+\tste_attr.send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS;\n+\tste_attr.send_attr.len = MLX5DR_WQE_SZ_GTA_CTRL + MLX5DR_WQE_SZ_GTA_DATA;\n+\tste_attr.gta_opcode = MLX5DR_WQE_GTA_OP_ACTIVATE;\n+\n+\t/* Fence first from previous depend WQEs */\n+\tste_attr.send_attr.fence = 1;\n+\n+\twhile (send_sq->head_dep_idx != send_sq->tail_dep_idx) {\n+\t\tdep_wqe = &send_sq->dep_wqe[send_sq->tail_dep_idx++ & (queue->num_entries - 1)];\n+\n+\t\t/* Notify HW on the last WQE */\n+\t\tste_attr.send_attr.notify_hw = (send_sq->tail_dep_idx == send_sq->head_dep_idx);\n+\t\tste_attr.send_attr.user_data = dep_wqe->user_data;\n+\t\tste_attr.send_attr.rule = dep_wqe->rule;\n+\n+\t\tste_attr.rtc_0 = dep_wqe->rtc_0;\n+\t\tste_attr.rtc_1 = dep_wqe->rtc_1;\n+\t\tste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0;\n+\t\tste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1;\n+\t\tste_attr.used_id_rtc_0 = &dep_wqe->rule->rtc_0;\n+\t\tste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1;\n+\t\tste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl;\n+\t\tste_attr.wqe_data = &dep_wqe->wqe_data;\n+\n+\t\tmlx5dr_send_ste(queue, &ste_attr);\n+\n+\t\t/* Fencing is done only on the first WQE */\n+\t\tste_attr.send_attr.fence = 0;\n+\t}\n+}\n+\n+struct mlx5dr_send_engine_post_ctrl\n+mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue)\n+{\n+\tstruct mlx5dr_send_engine_post_ctrl ctrl;\n+\n+\tctrl.queue = queue;\n+\t/* Currently only one send ring is supported */\n+\tctrl.send_ring = &queue->send_ring[0];\n+\tctrl.num_wqebbs = 0;\n+\n+\treturn ctrl;\n+}\n+\n+void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl,\n+\t\t\t\t char **buf, size_t *len)\n+{\n+\tstruct mlx5dr_send_ring_sq *send_sq = &ctrl->send_ring->send_sq;\n+\tunsigned int idx;\n+\n+\tidx = (send_sq->cur_post + ctrl->num_wqebbs) & send_sq->buf_mask;\n+\n+\t*buf = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT);\n+\t*len = MLX5_SEND_WQE_BB;\n+\n+\tif (!ctrl->num_wqebbs) {\n+\t\t*buf += sizeof(struct mlx5dr_wqe_ctrl_seg);\n+\t\t*len -= sizeof(struct mlx5dr_wqe_ctrl_seg);\n+\t}\n+\n+\tctrl->num_wqebbs++;\n+}\n+\n+static void mlx5dr_send_engine_post_ring(struct mlx5dr_send_ring_sq *sq,\n+\t\t\t\t\t struct mlx5dv_devx_uar *uar,\n+\t\t\t\t\t struct mlx5dr_wqe_ctrl_seg *wqe_ctrl)\n+{\n+\trte_compiler_barrier();\n+\tsq->db[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->cur_post);\n+\n+\trte_wmb();\n+\tmlx5dr_uar_write64_relaxed(*((uint64_t *)wqe_ctrl), uar->reg_addr);\n+\trte_wmb();\n+}\n+\n+static void\n+mlx5dr_send_wqe_set_tag(struct mlx5dr_wqe_gta_data_seg_ste *wqe_data,\n+\t\t\tstruct mlx5dr_rule_match_tag *tag,\n+\t\t\tbool is_jumbo)\n+{\n+\tif (is_jumbo) {\n+\t\t/* Clear previous possibly dirty control */\n+\t\tmemset(wqe_data, 0, MLX5DR_STE_CTRL_SZ);\n+\t\tmemcpy(wqe_data->action, tag->jumbo, MLX5DR_JUMBO_TAG_SZ);\n+\t} else {\n+\t\t/* Clear previous possibly dirty control and actions */\n+\t\tmemset(wqe_data, 0, MLX5DR_STE_CTRL_SZ + MLX5DR_ACTIONS_SZ);\n+\t\tmemcpy(wqe_data->tag, tag->match, MLX5DR_MATCH_TAG_SZ);\n+\t}\n+}\n+\n+void mlx5dr_send_engine_post_end(struct mlx5dr_send_engine_post_ctrl *ctrl,\n+\t\t\t\t struct mlx5dr_send_engine_post_attr *attr)\n+{\n+\tstruct mlx5dr_wqe_ctrl_seg *wqe_ctrl;\n+\tstruct mlx5dr_send_ring_sq *sq;\n+\tuint32_t flags = 0;\n+\tunsigned int idx;\n+\n+\tsq = &ctrl->send_ring->send_sq;\n+\tidx = sq->cur_post & sq->buf_mask;\n+\tsq->last_idx = idx;\n+\n+\twqe_ctrl = (void *)(sq->buf + (idx << MLX5_SEND_WQE_SHIFT));\n+\n+\twqe_ctrl->opmod_idx_opcode =\n+\t\trte_cpu_to_be_32((attr->opmod << 24) |\n+\t\t\t\t ((sq->cur_post & 0xffff) << 8) |\n+\t\t\t\t attr->opcode);\n+\twqe_ctrl->qpn_ds =\n+\t\trte_cpu_to_be_32((attr->len + sizeof(struct mlx5dr_wqe_ctrl_seg)) / 16 |\n+\t\t\t\t sq->sqn << 8);\n+\n+\twqe_ctrl->imm = rte_cpu_to_be_32(attr->id);\n+\n+\tflags |= attr->notify_hw ? MLX5_WQE_CTRL_CQ_UPDATE : 0;\n+\tflags |= attr->fence ? MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE : 0;\n+\twqe_ctrl->flags = rte_cpu_to_be_32(flags);\n+\n+\tsq->wr_priv[idx].id = attr->id;\n+\tsq->wr_priv[idx].retry_id = attr->retry_id;\n+\n+\tsq->wr_priv[idx].rule = attr->rule;\n+\tsq->wr_priv[idx].user_data = attr->user_data;\n+\tsq->wr_priv[idx].num_wqebbs = ctrl->num_wqebbs;\n+\n+\tif (attr->rule) {\n+\t\tsq->wr_priv[idx].rule->pending_wqes++;\n+\t\tsq->wr_priv[idx].used_id = attr->used_id;\n+\t}\n+\n+\tsq->cur_post += ctrl->num_wqebbs;\n+\n+\tif (attr->notify_hw)\n+\t\tmlx5dr_send_engine_post_ring(sq, ctrl->queue->uar, wqe_ctrl);\n+}\n+\n+static void mlx5dr_send_wqe(struct mlx5dr_send_engine *queue,\n+\t\t\t struct mlx5dr_send_engine_post_attr *send_attr,\n+\t\t\t struct mlx5dr_wqe_gta_ctrl_seg *send_wqe_ctrl,\n+\t\t\t void *send_wqe_data,\n+\t\t\t void *send_wqe_tag,\n+\t\t\t bool is_jumbo,\n+\t\t\t uint8_t gta_opcode,\n+\t\t\t uint32_t direct_index)\n+{\n+\tstruct mlx5dr_wqe_gta_data_seg_ste *wqe_data;\n+\tstruct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl;\n+\tstruct mlx5dr_send_engine_post_ctrl ctrl;\n+\tsize_t wqe_len;\n+\n+\tctrl = mlx5dr_send_engine_post_start(queue);\n+\tmlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_ctrl, &wqe_len);\n+\tmlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_data, &wqe_len);\n+\n+\twqe_ctrl->op_dirix = htobe32(gta_opcode << 28 | direct_index);\n+\tmemcpy(wqe_ctrl->stc_ix, send_wqe_ctrl->stc_ix, sizeof(send_wqe_ctrl->stc_ix));\n+\n+\tif (send_wqe_data)\n+\t\tmemcpy(wqe_data, send_wqe_data, sizeof(*wqe_data));\n+\telse\n+\t\tmlx5dr_send_wqe_set_tag(wqe_data, send_wqe_tag, is_jumbo);\n+\n+\tmlx5dr_send_engine_post_end(&ctrl, send_attr);\n+}\n+\n+void mlx5dr_send_ste(struct mlx5dr_send_engine *queue,\n+\t\t struct mlx5dr_send_ste_attr *ste_attr)\n+{\n+\tstruct mlx5dr_send_engine_post_attr *send_attr = &ste_attr->send_attr;\n+\tuint8_t notify_hw = send_attr->notify_hw;\n+\tuint8_t fence = send_attr->fence;\n+\n+\tif (ste_attr->rtc_1) {\n+\t\tsend_attr->id = ste_attr->rtc_1;\n+\t\tsend_attr->used_id = ste_attr->used_id_rtc_1;\n+\t\tsend_attr->retry_id = ste_attr->retry_rtc_1;\n+\t\tsend_attr->fence = fence;\n+\t\tsend_attr->notify_hw = notify_hw && !ste_attr->rtc_0;\n+\t\tmlx5dr_send_wqe(queue, send_attr,\n+\t\t\t\tste_attr->wqe_ctrl,\n+\t\t\t\tste_attr->wqe_data,\n+\t\t\t\tste_attr->wqe_tag,\n+\t\t\t\tste_attr->wqe_tag_is_jumbo,\n+\t\t\t\tste_attr->gta_opcode,\n+\t\t\t\tste_attr->direct_index);\n+\t}\n+\n+\tif (ste_attr->rtc_0) {\n+\t\tsend_attr->id = ste_attr->rtc_0;\n+\t\tsend_attr->used_id = ste_attr->used_id_rtc_0;\n+\t\tsend_attr->retry_id = ste_attr->retry_rtc_0;\n+\t\tsend_attr->fence = fence && !ste_attr->rtc_1;\n+\t\tsend_attr->notify_hw = notify_hw;\n+\t\tmlx5dr_send_wqe(queue, send_attr,\n+\t\t\t\tste_attr->wqe_ctrl,\n+\t\t\t\tste_attr->wqe_data,\n+\t\t\t\tste_attr->wqe_tag,\n+\t\t\t\tste_attr->wqe_tag_is_jumbo,\n+\t\t\t\tste_attr->gta_opcode,\n+\t\t\t\tste_attr->direct_index);\n+\t}\n+\n+\t/* Restore to ortginal requested values */\n+\tsend_attr->notify_hw = notify_hw;\n+\tsend_attr->fence = fence;\n+}\n+\n+static void mlx5dr_send_engine_retry_post_send(struct mlx5dr_send_engine *queue,\n+\t\t\t\t\t struct mlx5dr_send_ring_priv *priv,\n+\t\t\t\t\t uint16_t wqe_cnt)\n+{\n+\tstruct mlx5dr_send_engine_post_attr send_attr = {0};\n+\tstruct mlx5dr_wqe_gta_data_seg_ste *wqe_data;\n+\tstruct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl;\n+\tstruct mlx5dr_send_engine_post_ctrl ctrl;\n+\tstruct mlx5dr_send_ring_sq *send_sq;\n+\tunsigned int idx;\n+\tsize_t wqe_len;\n+\tchar *p;\n+\n+\tsend_attr.rule = priv->rule;\n+\tsend_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS;\n+\tsend_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE;\n+\tsend_attr.len = MLX5_SEND_WQE_BB * 2 - sizeof(struct mlx5dr_wqe_ctrl_seg);\n+\tsend_attr.notify_hw = 1;\n+\tsend_attr.fence = 0;\n+\tsend_attr.user_data = priv->user_data;\n+\tsend_attr.id = priv->retry_id;\n+\tsend_attr.used_id = priv->used_id;\n+\n+\tctrl = mlx5dr_send_engine_post_start(queue);\n+\tmlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_ctrl, &wqe_len);\n+\tmlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_data, &wqe_len);\n+\n+\tsend_sq = &ctrl.send_ring->send_sq;\n+\tidx = wqe_cnt & send_sq->buf_mask;\n+\tp = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT);\n+\n+\t/* Copy old gta ctrl */\n+\tmemcpy(wqe_ctrl, p + sizeof(struct mlx5dr_wqe_ctrl_seg),\n+\t MLX5_SEND_WQE_BB - sizeof(struct mlx5dr_wqe_ctrl_seg));\n+\n+\tidx = (wqe_cnt + 1) & send_sq->buf_mask;\n+\tp = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT);\n+\n+\t/* Copy old gta data */\n+\tmemcpy(wqe_data, p, MLX5_SEND_WQE_BB);\n+\n+\tmlx5dr_send_engine_post_end(&ctrl, &send_attr);\n+}\n+\n+void mlx5dr_send_engine_flush_queue(struct mlx5dr_send_engine *queue)\n+{\n+\tstruct mlx5dr_send_ring_sq *sq = &queue->send_ring[0].send_sq;\n+\tstruct mlx5dr_wqe_ctrl_seg *wqe_ctrl;\n+\n+\twqe_ctrl = (void *)(sq->buf + (sq->last_idx << MLX5_SEND_WQE_SHIFT));\n+\n+\twqe_ctrl->flags |= rte_cpu_to_be_32(MLX5_WQE_CTRL_CQ_UPDATE);\n+\n+\tmlx5dr_send_engine_post_ring(sq, queue->uar, wqe_ctrl);\n+}\n+\n+static void mlx5dr_send_engine_update_rule(struct mlx5dr_send_engine *queue,\n+\t\t\t\t\t struct mlx5dr_send_ring_priv *priv,\n+\t\t\t\t\t uint16_t wqe_cnt,\n+\t\t\t\t\t enum rte_flow_op_status *status)\n+{\n+\tpriv->rule->pending_wqes--;\n+\n+\tif (*status == RTE_FLOW_OP_ERROR) {\n+\t\tif (priv->retry_id) {\n+\t\t\tmlx5dr_send_engine_retry_post_send(queue, priv, wqe_cnt);\n+\t\t\treturn;\n+\t\t}\n+\t\t/* Some part of the rule failed */\n+\t\tpriv->rule->status = MLX5DR_RULE_STATUS_FAILING;\n+\t\t*priv->used_id = 0;\n+\t} else {\n+\t\t*priv->used_id = priv->id;\n+\t}\n+\n+\t/* Update rule status for the last completion */\n+\tif (!priv->rule->pending_wqes) {\n+\t\tif (unlikely(priv->rule->status == MLX5DR_RULE_STATUS_FAILING)) {\n+\t\t\t/* Rule completely failed and doesn't require cleanup */\n+\t\t\tif (!priv->rule->rtc_0 && !priv->rule->rtc_1)\n+\t\t\t\tpriv->rule->status = MLX5DR_RULE_STATUS_FAILED;\n+\n+\t\t\t*status = RTE_FLOW_OP_ERROR;\n+\t\t} else {\n+\t\t\t/* Increase the status, this only works on good flow as the enum\n+\t\t\t * is arrange it away creating -> created -> deleting -> deleted\n+\t\t\t */\n+\t\t\tpriv->rule->status++;\n+\t\t\t*status = RTE_FLOW_OP_SUCCESS;\n+\t\t\t/* Rule was deleted now we can safely release action STEs */\n+\t\t\tif (priv->rule->status == MLX5DR_RULE_STATUS_DELETED)\n+\t\t\t\tmlx5dr_rule_free_action_ste_idx(priv->rule);\n+\t\t}\n+\t}\n+}\n+\n+static void mlx5dr_send_engine_update(struct mlx5dr_send_engine *queue,\n+\t\t\t\t struct mlx5_cqe64 *cqe,\n+\t\t\t\t struct mlx5dr_send_ring_priv *priv,\n+\t\t\t\t struct rte_flow_op_result res[],\n+\t\t\t\t int64_t *i,\n+\t\t\t\t uint32_t res_nb,\n+\t\t\t\t uint16_t wqe_cnt)\n+{\n+\tenum rte_flow_op_status status;\n+\n+\tif (!cqe || (likely(rte_be_to_cpu_32(cqe->byte_cnt) >> 31 == 0) &&\n+\t likely(mlx5dv_get_cqe_opcode(cqe) == MLX5_CQE_REQ))) {\n+\t\tstatus = RTE_FLOW_OP_SUCCESS;\n+\t} else {\n+\t\tstatus = RTE_FLOW_OP_ERROR;\n+\t}\n+\n+\tif (priv->user_data) {\n+\t\tif (priv->rule) {\n+\t\t\tmlx5dr_send_engine_update_rule(queue, priv, wqe_cnt, &status);\n+\t\t\t/* Completion is provided on the last rule WQE */\n+\t\t\tif (priv->rule->pending_wqes)\n+\t\t\t\treturn;\n+\t\t}\n+\n+\t\tif (*i < res_nb) {\n+\t\t\tres[*i].user_data = priv->user_data;\n+\t\t\tres[*i].status = status;\n+\t\t\t(*i)++;\n+\t\t\tmlx5dr_send_engine_dec_rule(queue);\n+\t\t} else {\n+\t\t\tmlx5dr_send_engine_gen_comp(queue, priv->user_data, status);\n+\t\t}\n+\t}\n+}\n+\n+static void mlx5dr_send_engine_poll_cq(struct mlx5dr_send_engine *queue,\n+\t\t\t\t struct mlx5dr_send_ring *send_ring,\n+\t\t\t\t struct rte_flow_op_result res[],\n+\t\t\t\t int64_t *i,\n+\t\t\t\t uint32_t res_nb)\n+{\n+\tstruct mlx5dr_send_ring_cq *cq = &send_ring->send_cq;\n+\tstruct mlx5dr_send_ring_sq *sq = &send_ring->send_sq;\n+\tuint32_t cq_idx = cq->cons_index & cq->ncqe_mask;\n+\tstruct mlx5dr_send_ring_priv *priv;\n+\tstruct mlx5_cqe64 *cqe;\n+\tuint32_t offset_cqe64;\n+\tuint8_t cqe_opcode;\n+\tuint8_t cqe_owner;\n+\tuint16_t wqe_cnt;\n+\tuint8_t sw_own;\n+\n+\toffset_cqe64 = RTE_CACHE_LINE_SIZE - sizeof(struct mlx5_cqe64);\n+\tcqe = (void *)(cq->buf + (cq_idx << cq->cqe_log_sz) + offset_cqe64);\n+\n+\tsw_own = (cq->cons_index & cq->ncqe) ? 1 : 0;\n+\tcqe_opcode = mlx5dv_get_cqe_opcode(cqe);\n+\tcqe_owner = mlx5dv_get_cqe_owner(cqe);\n+\n+\tif (cqe_opcode == MLX5_CQE_INVALID ||\n+\t cqe_owner != sw_own)\n+\t\treturn;\n+\n+\tif (unlikely(mlx5dv_get_cqe_opcode(cqe) != MLX5_CQE_REQ))\n+\t\tqueue->err = true;\n+\n+\trte_io_rmb();\n+\n+\twqe_cnt = be16toh(cqe->wqe_counter) & sq->buf_mask;\n+\n+\twhile (cq->poll_wqe != wqe_cnt) {\n+\t\tpriv = &sq->wr_priv[cq->poll_wqe];\n+\t\tmlx5dr_send_engine_update(queue, NULL, priv, res, i, res_nb, 0);\n+\t\tcq->poll_wqe = (cq->poll_wqe + priv->num_wqebbs) & sq->buf_mask;\n+\t}\n+\n+\tpriv = &sq->wr_priv[wqe_cnt];\n+\tcq->poll_wqe = (wqe_cnt + priv->num_wqebbs) & sq->buf_mask;\n+\tmlx5dr_send_engine_update(queue, cqe, priv, res, i, res_nb, wqe_cnt);\n+\tcq->cons_index++;\n+}\n+\n+static void mlx5dr_send_engine_poll_cqs(struct mlx5dr_send_engine *queue,\n+\t\t\t\t\tstruct rte_flow_op_result res[],\n+\t\t\t\t\tint64_t *polled,\n+\t\t\t\t\tuint32_t res_nb)\n+{\n+\tint j;\n+\n+\tfor (j = 0; j < MLX5DR_NUM_SEND_RINGS; j++) {\n+\t\tmlx5dr_send_engine_poll_cq(queue, &queue->send_ring[j],\n+\t\t\t\t\t res, polled, res_nb);\n+\n+\t\t*queue->send_ring[j].send_cq.db =\n+\t\t\thtobe32(queue->send_ring[j].send_cq.cons_index & 0xffffff);\n+\t}\n+}\n+\n+static void mlx5dr_send_engine_poll_list(struct mlx5dr_send_engine *queue,\n+\t\t\t\t\t struct rte_flow_op_result res[],\n+\t\t\t\t\t int64_t *polled,\n+\t\t\t\t\t uint32_t res_nb)\n+{\n+\tstruct mlx5dr_completed_poll *comp = &queue->completed;\n+\n+\twhile (comp->ci != comp->pi) {\n+\t\tif (*polled < res_nb) {\n+\t\t\tres[*polled].status =\n+\t\t\t\tcomp->entries[comp->ci].status;\n+\t\t\tres[*polled].user_data =\n+\t\t\t\tcomp->entries[comp->ci].user_data;\n+\t\t\t(*polled)++;\n+\t\t\tcomp->ci = (comp->ci + 1) & comp->mask;\n+\t\t\tmlx5dr_send_engine_dec_rule(queue);\n+\t\t} else {\n+\t\t\treturn;\n+\t\t}\n+\t}\n+}\n+\n+static int mlx5dr_send_engine_poll(struct mlx5dr_send_engine *queue,\n+\t\t\t\t struct rte_flow_op_result res[],\n+\t\t\t\t uint32_t res_nb)\n+{\n+\tint64_t polled = 0;\n+\n+\tmlx5dr_send_engine_poll_list(queue, res, &polled, res_nb);\n+\n+\tif (polled >= res_nb)\n+\t\treturn polled;\n+\n+\tmlx5dr_send_engine_poll_cqs(queue, res, &polled, res_nb);\n+\n+\treturn polled;\n+}\n+\n+int mlx5dr_send_queue_poll(struct mlx5dr_context *ctx,\n+\t\t\t uint16_t queue_id,\n+\t\t\t struct rte_flow_op_result res[],\n+\t\t\t uint32_t res_nb)\n+{\n+\treturn mlx5dr_send_engine_poll(&ctx->send_queue[queue_id],\n+\t\t\t\t res, res_nb);\n+}\n+\n+static int mlx5dr_send_ring_create_sq_obj(struct mlx5dr_context *ctx,\n+\t\t\t\t\t struct mlx5dr_send_engine *queue,\n+\t\t\t\t\t struct mlx5dr_send_ring_sq *sq,\n+\t\t\t\t\t struct mlx5dr_send_ring_cq *cq,\n+\t\t\t\t\t size_t log_wq_sz)\n+{\n+\tstruct mlx5dr_cmd_sq_create_attr attr = {0};\n+\tint err;\n+\n+\tattr.cqn = cq->cqn;\n+\tattr.pdn = ctx->pd_num;\n+\tattr.page_id = queue->uar->page_id;\n+\tattr.dbr_id = sq->db_umem->umem_id;\n+\tattr.wq_id = sq->buf_umem->umem_id;\n+\tattr.log_wq_sz = log_wq_sz;\n+\n+\tsq->obj = mlx5dr_cmd_sq_create(ctx->ibv_ctx, &attr);\n+\tif (!sq->obj)\n+\t\treturn rte_errno;\n+\n+\tsq->sqn = sq->obj->id;\n+\n+\terr = mlx5dr_cmd_sq_modify_rdy(sq->obj);\n+\tif (err)\n+\t\tgoto free_sq;\n+\n+\treturn 0;\n+\n+free_sq:\n+\tmlx5dr_cmd_destroy_obj(sq->obj);\n+\n+\treturn err;\n+}\n+\n+static inline unsigned long align(unsigned long val, unsigned long align)\n+{\n+\treturn (val + align - 1) & ~(align - 1);\n+}\n+\n+static int mlx5dr_send_ring_open_sq(struct mlx5dr_context *ctx,\n+\t\t\t\t struct mlx5dr_send_engine *queue,\n+\t\t\t\t struct mlx5dr_send_ring_sq *sq,\n+\t\t\t\t struct mlx5dr_send_ring_cq *cq)\n+{\n+\tsize_t sq_log_buf_sz;\n+\tsize_t buf_aligned;\n+\tsize_t sq_buf_sz;\n+\tsize_t buf_sz;\n+\tint err;\n+\n+\tbuf_sz = queue->num_entries * MAX_WQES_PER_RULE;\n+\tsq_log_buf_sz = log2above(buf_sz);\n+\tsq_buf_sz = 1 << (sq_log_buf_sz + log2above(MLX5_SEND_WQE_BB));\n+\tsq->reg_addr = queue->uar->reg_addr;\n+\n+\tbuf_aligned = align(sq_buf_sz, sysconf(_SC_PAGESIZE));\n+\terr = posix_memalign((void **)&sq->buf, sysconf(_SC_PAGESIZE), buf_aligned);\n+\tif (err) {\n+\t\trte_errno = ENOMEM;\n+\t\treturn err;\n+\t}\n+\tmemset(sq->buf, 0, buf_aligned);\n+\n+\terr = posix_memalign((void **)&sq->db, 8, 8);\n+\tif (err)\n+\t\tgoto free_buf;\n+\n+\tsq->buf_umem = mlx5_glue->devx_umem_reg(ctx->ibv_ctx, sq->buf, sq_buf_sz, 0);\n+\n+\tif (!sq->buf_umem) {\n+\t\terr = errno;\n+\t\tgoto free_db;\n+\t}\n+\n+\tsq->db_umem = mlx5_glue->devx_umem_reg(ctx->ibv_ctx, sq->db, 8, 0);\n+\tif (!sq->db_umem) {\n+\t\terr = errno;\n+\t\tgoto free_buf_umem;\n+\t}\n+\n+\terr = mlx5dr_send_ring_create_sq_obj(ctx, queue, sq, cq, sq_log_buf_sz);\n+\n+\tif (err)\n+\t\tgoto free_db_umem;\n+\n+\tsq->wr_priv = simple_malloc(sizeof(*sq->wr_priv) * buf_sz);\n+\tif (!sq->wr_priv) {\n+\t\terr = ENOMEM;\n+\t\tgoto destroy_sq_obj;\n+\t}\n+\n+\tsq->dep_wqe = simple_calloc(queue->num_entries, sizeof(*sq->dep_wqe));\n+\tif (!sq->dep_wqe) {\n+\t\terr = ENOMEM;\n+\t\tgoto destroy_wr_priv;\n+\t}\n+\n+\tsq->buf_mask = buf_sz - 1;\n+\n+\treturn 0;\n+\n+destroy_wr_priv:\n+\tsimple_free(sq->wr_priv);\n+destroy_sq_obj:\n+\tmlx5dr_cmd_destroy_obj(sq->obj);\n+free_db_umem:\n+\tmlx5_glue->devx_umem_dereg(sq->db_umem);\n+free_buf_umem:\n+\tmlx5_glue->devx_umem_dereg(sq->buf_umem);\n+free_db:\n+\tfree(sq->db);\n+free_buf:\n+\tfree(sq->buf);\n+\trte_errno = err;\n+\treturn err;\n+}\n+\n+static void mlx5dr_send_ring_close_sq(struct mlx5dr_send_ring_sq *sq)\n+{\n+\tsimple_free(sq->dep_wqe);\n+\tmlx5dr_cmd_destroy_obj(sq->obj);\n+\tmlx5_glue->devx_umem_dereg(sq->db_umem);\n+\tmlx5_glue->devx_umem_dereg(sq->buf_umem);\n+\tsimple_free(sq->wr_priv);\n+\tfree(sq->db);\n+\tfree(sq->buf);\n+}\n+\n+static int mlx5dr_send_ring_open_cq(struct mlx5dr_context *ctx,\n+\t\t\t\t struct mlx5dr_send_engine *queue,\n+\t\t\t\t struct mlx5dr_send_ring_cq *cq)\n+{\n+\tstruct mlx5dv_cq mlx5_cq = {0};\n+\tstruct mlx5dv_obj obj;\n+\tstruct ibv_cq *ibv_cq;\n+\tsize_t cq_size;\n+\tint err;\n+\n+\tcq_size = queue->num_entries;\n+\tibv_cq = mlx5_glue->create_cq(ctx->ibv_ctx, cq_size, NULL, NULL, 0);\n+\tif (!ibv_cq) {\n+\t\tDR_LOG(ERR, \"Failed to create CQ\");\n+\t\trte_errno = errno;\n+\t\treturn rte_errno;\n+\t}\n+\n+\tobj.cq.in = ibv_cq;\n+\tobj.cq.out = &mlx5_cq;\n+\terr = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ);\n+\tif (err) {\n+\t\terr = errno;\n+\t\tgoto close_cq;\n+\t}\n+\n+\tcq->buf = mlx5_cq.buf;\n+\tcq->db = mlx5_cq.dbrec;\n+\tcq->ncqe = mlx5_cq.cqe_cnt;\n+\tcq->cqe_sz = mlx5_cq.cqe_size;\n+\tcq->cqe_log_sz = log2above(cq->cqe_sz);\n+\tcq->ncqe_mask = cq->ncqe - 1;\n+\tcq->buf_sz = cq->cqe_sz * cq->ncqe;\n+\tcq->cqn = mlx5_cq.cqn;\n+\tcq->ibv_cq = ibv_cq;\n+\n+\treturn 0;\n+\n+close_cq:\n+\tmlx5_glue->destroy_cq(ibv_cq);\n+\trte_errno = err;\n+\treturn err;\n+}\n+\n+static void mlx5dr_send_ring_close_cq(struct mlx5dr_send_ring_cq *cq)\n+{\n+\tmlx5_glue->destroy_cq(cq->ibv_cq);\n+}\n+\n+static void mlx5dr_send_ring_close(struct mlx5dr_send_ring *ring)\n+{\n+\tmlx5dr_send_ring_close_sq(&ring->send_sq);\n+\tmlx5dr_send_ring_close_cq(&ring->send_cq);\n+}\n+\n+static int mlx5dr_send_ring_open(struct mlx5dr_context *ctx,\n+\t\t\t\t struct mlx5dr_send_engine *queue,\n+\t\t\t\t struct mlx5dr_send_ring *ring)\n+{\n+\tint err;\n+\n+\terr = mlx5dr_send_ring_open_cq(ctx, queue, &ring->send_cq);\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = mlx5dr_send_ring_open_sq(ctx, queue, &ring->send_sq, &ring->send_cq);\n+\tif (err)\n+\t\tgoto close_cq;\n+\n+\treturn err;\n+\n+close_cq:\n+\tmlx5dr_send_ring_close_cq(&ring->send_cq);\n+\n+\treturn err;\n+}\n+\n+static void __mlx5dr_send_rings_close(struct mlx5dr_send_engine *queue,\n+\t\t\t\t uint16_t i)\n+{\n+\twhile (i--)\n+\t\tmlx5dr_send_ring_close(&queue->send_ring[i]);\n+}\n+\n+static void mlx5dr_send_rings_close(struct mlx5dr_send_engine *queue)\n+{\n+\t__mlx5dr_send_rings_close(queue, queue->rings);\n+}\n+\n+static int mlx5dr_send_rings_open(struct mlx5dr_context *ctx,\n+\t\t\t\t struct mlx5dr_send_engine *queue)\n+{\n+\tuint16_t i;\n+\tint err;\n+\n+\tfor (i = 0; i < queue->rings; i++) {\n+\t\terr = mlx5dr_send_ring_open(ctx, queue, &queue->send_ring[i]);\n+\t\tif (err)\n+\t\t\tgoto free_rings;\n+\t}\n+\n+\treturn 0;\n+\n+free_rings:\n+\t__mlx5dr_send_rings_close(queue, i);\n+\n+\treturn err;\n+}\n+\n+void mlx5dr_send_queue_close(struct mlx5dr_send_engine *queue)\n+{\n+\tmlx5dr_send_rings_close(queue);\n+\tsimple_free(queue->completed.entries);\n+\tmlx5_glue->devx_free_uar(queue->uar);\n+}\n+\n+int mlx5dr_send_queue_open(struct mlx5dr_context *ctx,\n+\t\t\t struct mlx5dr_send_engine *queue,\n+\t\t\t uint16_t queue_size)\n+{\n+\tstruct mlx5dv_devx_uar *uar;\n+\tint err;\n+\n+#ifdef MLX5DV_UAR_ALLOC_TYPE_NC\n+\tuar = mlx5_glue->devx_alloc_uar(ctx->ibv_ctx, MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC);\n+\tif (!uar) {\n+\t\trte_errno = errno;\n+\t\treturn rte_errno;\n+\t}\n+#else\n+\tuar = NULL;\n+\trte_errno = ENOTSUP;\n+\treturn rte_errno;\n+#endif\n+\n+\tqueue->uar = uar;\n+\tqueue->rings = MLX5DR_NUM_SEND_RINGS;\n+\tqueue->num_entries = roundup_pow_of_two(queue_size);\n+\tqueue->used_entries = 0;\n+\tqueue->th_entries = queue->num_entries;\n+\n+\tqueue->completed.entries = simple_calloc(queue->num_entries,\n+\t\t\t\t\t\t sizeof(queue->completed.entries[0]));\n+\tif (!queue->completed.entries) {\n+\t\trte_errno = ENOMEM;\n+\t\tgoto free_uar;\n+\t}\n+\tqueue->completed.pi = 0;\n+\tqueue->completed.ci = 0;\n+\tqueue->completed.mask = queue->num_entries - 1;\n+\n+\terr = mlx5dr_send_rings_open(ctx, queue);\n+\tif (err)\n+\t\tgoto free_completed_entries;\n+\n+\treturn 0;\n+\n+free_completed_entries:\n+\tsimple_free(queue->completed.entries);\n+free_uar:\n+\tmlx5_glue->devx_free_uar(uar);\n+\treturn rte_errno;\n+}\n+\n+static void __mlx5dr_send_queues_close(struct mlx5dr_context *ctx, uint16_t queues)\n+{\n+\tstruct mlx5dr_send_engine *queue;\n+\n+\twhile (queues--) {\n+\t\tqueue = &ctx->send_queue[queues];\n+\n+\t\tmlx5dr_send_queue_close(queue);\n+\t}\n+}\n+\n+void mlx5dr_send_queues_close(struct mlx5dr_context *ctx)\n+{\n+\t__mlx5dr_send_queues_close(ctx, ctx->queues);\n+\tsimple_free(ctx->send_queue);\n+}\n+\n+int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,\n+\t\t\t uint16_t queues,\n+\t\t\t uint16_t queue_size)\n+{\n+\tint err = 0;\n+\tuint32_t i;\n+\n+\t/* Open one extra queue for control path */\n+\tctx->queues = queues + 1;\n+\n+\tctx->send_queue = simple_calloc(ctx->queues, sizeof(*ctx->send_queue));\n+\tif (!ctx->send_queue) {\n+\t\trte_errno = ENOMEM;\n+\t\treturn rte_errno;\n+\t}\n+\n+\tfor (i = 0; i < ctx->queues; i++) {\n+\t\terr = mlx5dr_send_queue_open(ctx, &ctx->send_queue[i], queue_size);\n+\t\tif (err)\n+\t\t\tgoto close_send_queues;\n+\t}\n+\n+\treturn 0;\n+\n+close_send_queues:\n+\t __mlx5dr_send_queues_close(ctx, i);\n+\n+\tsimple_free(ctx->send_queue);\n+\n+\treturn err;\n+}\n+\n+int mlx5dr_send_queue_action(struct mlx5dr_context *ctx,\n+\t\t\t uint16_t queue_id,\n+\t\t\t uint32_t actions)\n+{\n+\tstruct mlx5dr_send_ring_sq *send_sq;\n+\tstruct mlx5dr_send_engine *queue;\n+\n+\tqueue = &ctx->send_queue[queue_id];\n+\tsend_sq = &queue->send_ring->send_sq;\n+\n+\tif (actions == MLX5DR_SEND_QUEUE_ACTION_DRAIN) {\n+\t\tif (send_sq->head_dep_idx != send_sq->tail_dep_idx)\n+\t\t\t/* Send dependent WQEs to drain the queue */\n+\t\t\tmlx5dr_send_all_dep_wqe(queue);\n+\t\telse\n+\t\t\t/* Signal on the last posted WQE */\n+\t\t\tmlx5dr_send_engine_flush_queue(queue);\n+\t} else {\n+\t\trte_errno = -EINVAL;\n+\t\treturn rte_errno;\n+\t}\n+\n+\treturn 0;\n+}\ndiff --git a/drivers/net/mlx5/hws/mlx5dr_send.h b/drivers/net/mlx5/hws/mlx5dr_send.h\nnew file mode 100644\nindex 0000000000..8d4769495d\n--- /dev/null\n+++ b/drivers/net/mlx5/hws/mlx5dr_send.h\n@@ -0,0 +1,275 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates\n+ */\n+\n+#ifndef MLX5DR_SEND_H_\n+#define MLX5DR_SEND_H_\n+\n+#define MLX5DR_NUM_SEND_RINGS 1\n+\n+/* As a single operation requires at least two WQEBBS.\n+ * This means a maximum of 16 such operations per rule.\n+ */\n+#define MAX_WQES_PER_RULE 32\n+\n+/* WQE Control segment. */\n+struct mlx5dr_wqe_ctrl_seg {\n+\t__be32 opmod_idx_opcode;\n+\t__be32 qpn_ds;\n+\t__be32 flags;\n+\t__be32 imm;\n+};\n+\n+enum mlx5dr_wqe_opcode {\n+\tMLX5DR_WQE_OPCODE_TBL_ACCESS = 0x2c,\n+};\n+\n+enum mlx5dr_wqe_opmod {\n+\tMLX5DR_WQE_OPMOD_GTA_STE = 0,\n+\tMLX5DR_WQE_OPMOD_GTA_MOD_ARG = 1,\n+};\n+\n+enum mlx5dr_wqe_gta_opcode {\n+\tMLX5DR_WQE_GTA_OP_ACTIVATE = 0,\n+\tMLX5DR_WQE_GTA_OP_DEACTIVATE = 1,\n+};\n+\n+enum mlx5dr_wqe_gta_opmod {\n+\tMLX5DR_WQE_GTA_OPMOD_STE = 0,\n+\tMLX5DR_WQE_GTA_OPMOD_MOD_ARG = 1,\n+};\n+\n+enum mlx5dr_wqe_gta_sz {\n+\tMLX5DR_WQE_SZ_GTA_CTRL = 48,\n+\tMLX5DR_WQE_SZ_GTA_DATA = 64,\n+};\n+\n+struct mlx5dr_wqe_gta_ctrl_seg {\n+\t__be32 op_dirix;\n+\t__be32 stc_ix[5];\n+\t__be32 rsvd0[6];\n+};\n+\n+struct mlx5dr_wqe_gta_data_seg_ste {\n+\t__be32 rsvd0_ctr_id;\n+\t__be32 rsvd1[4];\n+\t__be32 action[3];\n+\t__be32 tag[8];\n+};\n+\n+struct mlx5dr_wqe_gta_data_seg_arg {\n+\t__be32 action_args[8];\n+};\n+\n+struct mlx5dr_wqe_gta {\n+\tstruct mlx5dr_wqe_gta_ctrl_seg gta_ctrl;\n+\tunion {\n+\t\tstruct mlx5dr_wqe_gta_data_seg_ste seg_ste;\n+\t\tstruct mlx5dr_wqe_gta_data_seg_arg seg_arg;\n+\t};\n+};\n+\n+struct mlx5dr_send_ring_cq {\n+\tuint8_t *buf;\n+\tuint32_t cons_index;\n+\tuint32_t ncqe_mask;\n+\tuint32_t buf_sz;\n+\tuint32_t ncqe;\n+\tuint32_t cqe_log_sz;\n+\t__be32 *db;\n+\tuint16_t poll_wqe;\n+\tstruct ibv_cq *ibv_cq;\n+\tuint32_t cqn;\n+\tuint32_t cqe_sz;\n+};\n+\n+struct mlx5dr_send_ring_priv {\n+\tstruct mlx5dr_rule *rule;\n+\tvoid *user_data;\n+\tuint32_t num_wqebbs;\n+\tuint32_t id;\n+\tuint32_t retry_id;\n+\tuint32_t *used_id;\n+};\n+\n+struct mlx5dr_send_ring_dep_wqe {\n+\tstruct mlx5dr_wqe_gta_ctrl_seg wqe_ctrl;\n+\tstruct mlx5dr_wqe_gta_data_seg_ste wqe_data;\n+\tstruct mlx5dr_rule *rule;\n+\tuint32_t rtc_0;\n+\tuint32_t rtc_1;\n+\tuint32_t retry_rtc_0;\n+\tuint32_t retry_rtc_1;\n+\tvoid *user_data;\n+};\n+\n+struct mlx5dr_send_ring_sq {\n+\tchar *buf;\n+\tuint32_t sqn;\n+\t__be32 *db;\n+\tvoid *reg_addr;\n+\tuint16_t cur_post;\n+\tuint16_t buf_mask;\n+\tstruct mlx5dr_send_ring_priv *wr_priv;\n+\tunsigned int last_idx;\n+\tstruct mlx5dr_send_ring_dep_wqe *dep_wqe;\n+\tunsigned int head_dep_idx;\n+\tunsigned int tail_dep_idx;\n+\tstruct mlx5dr_devx_obj *obj;\n+\tstruct mlx5dv_devx_umem *buf_umem;\n+\tstruct mlx5dv_devx_umem *db_umem;\n+};\n+\n+struct mlx5dr_send_ring {\n+\tstruct mlx5dr_send_ring_cq send_cq;\n+\tstruct mlx5dr_send_ring_sq send_sq;\n+};\n+\n+struct mlx5dr_completed_poll_entry {\n+\tvoid *user_data;\n+\tenum rte_flow_op_status status;\n+};\n+\n+struct mlx5dr_completed_poll {\n+\tstruct mlx5dr_completed_poll_entry *entries;\n+\tuint16_t ci;\n+\tuint16_t pi;\n+\tuint16_t mask;\n+};\n+\n+struct mlx5dr_send_engine {\n+\tstruct mlx5dr_send_ring send_ring[MLX5DR_NUM_SEND_RINGS]; /* For now 1:1 mapping */\n+\tstruct mlx5dv_devx_uar *uar; /* Uar is shared between rings of a queue */\n+\tstruct mlx5dr_completed_poll completed;\n+\tuint16_t used_entries;\n+\tuint16_t th_entries;\n+\tuint16_t rings;\n+\tuint16_t num_entries;\n+\tbool err;\n+} __rte_cache_aligned;\n+\n+struct mlx5dr_send_engine_post_ctrl {\n+\tstruct mlx5dr_send_engine *queue;\n+\tstruct mlx5dr_send_ring *send_ring;\n+\tsize_t num_wqebbs;\n+};\n+\n+struct mlx5dr_send_engine_post_attr {\n+\tuint8_t opcode;\n+\tuint8_t opmod;\n+\tuint8_t notify_hw;\n+\tuint8_t fence;\n+\tsize_t len;\n+\tstruct mlx5dr_rule *rule;\n+\tuint32_t id;\n+\tuint32_t retry_id;\n+\tuint32_t *used_id;\n+\tvoid *user_data;\n+};\n+\n+struct mlx5dr_send_ste_attr {\n+\t/* rtc / retry_rtc / used_id_rtc override send_attr */\n+\tuint32_t rtc_0;\n+\tuint32_t rtc_1;\n+\tuint32_t retry_rtc_0;\n+\tuint32_t retry_rtc_1;\n+\tuint32_t *used_id_rtc_0;\n+\tuint32_t *used_id_rtc_1;\n+\tbool wqe_tag_is_jumbo;\n+\tuint8_t gta_opcode;\n+\tuint32_t direct_index;\n+\tstruct mlx5dr_send_engine_post_attr send_attr;\n+\tstruct mlx5dr_rule_match_tag *wqe_tag;\n+\tstruct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl;\n+\tstruct mlx5dr_wqe_gta_data_seg_ste *wqe_data;\n+};\n+\n+/**\n+ * Provide safe 64bit store operation to mlx5 UAR region for\n+ * both 32bit and 64bit architectures.\n+ *\n+ * @param val\n+ * value to write in CPU endian format.\n+ * @param addr\n+ * Address to write to.\n+ * @param lock\n+ * Address of the lock to use for that UAR access.\n+ */\n+static __rte_always_inline void\n+mlx5dr_uar_write64_relaxed(uint64_t val, void *addr)\n+{\n+#ifdef RTE_ARCH_64\n+\t*(uint64_t *)addr = val;\n+#else /* !RTE_ARCH_64 */\n+\t*(uint32_t *)addr = val;\n+\trte_io_wmb();\n+\t*((uint32_t *)addr + 1) = val >> 32;\n+#endif\n+}\n+\n+struct mlx5dr_send_ring_dep_wqe *\n+mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue);\n+\n+void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue);\n+\n+void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue);\n+\n+void mlx5dr_send_queue_close(struct mlx5dr_send_engine *queue);\n+\n+int mlx5dr_send_queue_open(struct mlx5dr_context *ctx,\n+\t\t\t struct mlx5dr_send_engine *queue,\n+\t\t\t uint16_t queue_size);\n+\n+void mlx5dr_send_queues_close(struct mlx5dr_context *ctx);\n+\n+int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,\n+\t\t\t uint16_t queues,\n+\t\t\t uint16_t queue_size);\n+\n+struct mlx5dr_send_engine_post_ctrl\n+mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue);\n+\n+void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl,\n+\t\t\t\t char **buf, size_t *len);\n+\n+void mlx5dr_send_engine_post_end(struct mlx5dr_send_engine_post_ctrl *ctrl,\n+\t\t\t\t struct mlx5dr_send_engine_post_attr *attr);\n+\n+void mlx5dr_send_ste(struct mlx5dr_send_engine *queue,\n+\t\t struct mlx5dr_send_ste_attr *ste_attr);\n+\n+void mlx5dr_send_engine_flush_queue(struct mlx5dr_send_engine *queue);\n+\n+static inline bool mlx5dr_send_engine_full(struct mlx5dr_send_engine *queue)\n+{\n+\treturn queue->used_entries >= queue->th_entries;\n+}\n+\n+static inline void mlx5dr_send_engine_inc_rule(struct mlx5dr_send_engine *queue)\n+{\n+\tqueue->used_entries++;\n+}\n+\n+static inline void mlx5dr_send_engine_dec_rule(struct mlx5dr_send_engine *queue)\n+{\n+\tqueue->used_entries--;\n+}\n+\n+static inline void mlx5dr_send_engine_gen_comp(struct mlx5dr_send_engine *queue,\n+\t\t\t\t\t void *user_data,\n+\t\t\t\t\t int comp_status)\n+{\n+\tstruct mlx5dr_completed_poll *comp = &queue->completed;\n+\n+\tcomp->entries[comp->pi].status = comp_status;\n+\tcomp->entries[comp->pi].user_data = user_data;\n+\n+\tcomp->pi = (comp->pi + 1) & comp->mask;\n+}\n+\n+static inline bool mlx5dr_send_engine_err(struct mlx5dr_send_engine *queue)\n+{\n+\treturn queue->err;\n+}\n+\n+#endif /* MLX5DR_SEND_H_ */\n", "prefixes": [ "v6", "10/18" ] }