Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 117217,
    "url": "http://patches.dpdk.org/api/patches/117217/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/20220930125315.5079-9-suanmingm@nvidia.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20220930125315.5079-9-suanmingm@nvidia.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20220930125315.5079-9-suanmingm@nvidia.com",
    "date": "2022-09-30T12:53:06",
    "name": "[v3,08/17] net/mlx5: add HW steering counter action",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "5991a246ab790ff3d5207906902eec14cee99467",
    "submitter": {
        "id": 1887,
        "url": "http://patches.dpdk.org/api/people/1887/?format=api",
        "name": "Suanming Mou",
        "email": "suanmingm@nvidia.com"
    },
    "delegate": {
        "id": 3268,
        "url": "http://patches.dpdk.org/api/users/3268/?format=api",
        "username": "rasland",
        "first_name": "Raslan",
        "last_name": "Darawsheh",
        "email": "rasland@nvidia.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/20220930125315.5079-9-suanmingm@nvidia.com/mbox/",
    "series": [
        {
            "id": 24935,
            "url": "http://patches.dpdk.org/api/series/24935/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=24935",
            "date": "2022-09-30T12:52:58",
            "name": "net/mlx5: HW steering PMD update",
            "version": 3,
            "mbox": "http://patches.dpdk.org/series/24935/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/117217/comments/",
    "check": "warning",
    "checks": "http://patches.dpdk.org/api/patches/117217/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 1A1DBA00C4;\n\tFri, 30 Sep 2022 14:54:35 +0200 (CEST)",
            "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 97FE642B76;\n\tFri, 30 Sep 2022 14:54:02 +0200 (CEST)",
            "from NAM10-MW2-obe.outbound.protection.outlook.com\n (mail-mw2nam10on2076.outbound.protection.outlook.com [40.107.94.76])\n by mails.dpdk.org (Postfix) with ESMTP id 08B2142B83\n for <dev@dpdk.org>; Fri, 30 Sep 2022 14:54:01 +0200 (CEST)",
            "from DS7PR03CA0143.namprd03.prod.outlook.com (2603:10b6:5:3b4::28)\n by CY5PR12MB6180.namprd12.prod.outlook.com (2603:10b6:930:23::8) with\n Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5676.17; Fri, 30 Sep\n 2022 12:53:56 +0000",
            "from DM6NAM11FT102.eop-nam11.prod.protection.outlook.com\n (2603:10b6:5:3b4:cafe::bf) by DS7PR03CA0143.outlook.office365.com\n (2603:10b6:5:3b4::28) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5676.23 via Frontend\n Transport; Fri, 30 Sep 2022 12:53:55 +0000",
            "from mail.nvidia.com (216.228.117.161) by\n DM6NAM11FT102.mail.protection.outlook.com (10.13.173.172) with Microsoft SMTP\n Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id\n 15.20.5676.17 via Frontend Transport; Fri, 30 Sep 2022 12:53:55 +0000",
            "from rnnvmail201.nvidia.com (10.129.68.8) by mail.nvidia.com\n (10.129.200.67) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.986.26; Fri, 30 Sep\n 2022 05:53:51 -0700",
            "from nvidia.com (10.126.230.35) by rnnvmail201.nvidia.com\n (10.129.68.8) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.986.29; Fri, 30 Sep\n 2022 05:53:49 -0700"
        ],
        "ARC-Seal": "i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none;\n b=VR9x9R5m0NyeKtxVoDMq89f35TfiImtiS8JUGJwqKiV4dEw1ugkFaAdo2ufL2qYoEeaOg9DWrBa46UIkSIsTofwuOwEAizrJhjLvifrXj54d36UWU/hEDiMp9FQayIOgJ6Elnif5Eg/+1WECID6IwDE1ZjTcxn2E6tiZB1KoL+VDuNDYuhNMvVudagXCnsQ6hV0QyoZJ4vSyHSHdrqNYPhZBB7fQ5+nmXwovaGg4PhV0sVnVf2+69AklVOLBzq8IQADqwKb5Zpxi4avO95NKaP/90xkC3mcOiUTYLz4ScNzCK4A/v7prus1ydu3RyWC685Yrt5TifYwuT6IqHvJJBw==",
        "ARC-Message-Signature": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com;\n s=arcselector9901;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1;\n bh=r2wmDWH0haKeT+5FY/6qI28oWAqoJrAQCuxxwgmxxG0=;\n b=dpCT9WdcQG2BpYJe+IQjivJdTqhf4z7pr9LFYsJDBbBXdl85E4d15OxbIoyHxIfTyJQ4haXwd1HU6K5pBTn9ZazjU5WaHwkbP7LK+zhrNdK+Pplfvjz9Mo3AIpx4MliTpgj//IcYxCyAsnrDoqRgXHoeSmRujxYNUEE/ugj5puntC+o0XWcbh4Bb/RVQmdjYy380rszklCEq2SOqrecvyNWHxYA4OW+kh1PcQL7N5SjCRg/GkhfM1/x3iBDXlFDDjAy5hDWZE8YkQVKob97r5wAH2+yFKW/LcT6RI9X8jM2jlZVOShb/NiBmZebFSSrSyunZ5tHhDhEJXKadY8q13A==",
        "ARC-Authentication-Results": "i=1; mx.microsoft.com 1; spf=pass (sender ip is\n 216.228.117.161) smtp.rcpttodomain=ashroe.eu smtp.mailfrom=nvidia.com;\n dmarc=pass (p=reject sp=reject pct=100) action=none header.from=nvidia.com;\n dkim=none (message not signed); arc=none",
        "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=Nvidia.com;\n s=selector2;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=r2wmDWH0haKeT+5FY/6qI28oWAqoJrAQCuxxwgmxxG0=;\n b=CxkB/M2fhYXID1qEaJoccT5MFgtWi9XCWaED7r/FRLHnrwgSQ3GzEjZTu+54XFDh2IYooQbRhOxIXfzXS0XggsIuIbJAJo/VeW0jE6knMHuu/c4ij7x9RCBnXeAsG+dis22gQIu1GCxO63jWTk1rAXD67gIgy4Qz61J573PbhUablIF7deWOIBxGMUe7VZhCDtP9TR0rfhc9jiq6D0JdFkAIDPmlO4IC9s1+OSzPuQkG98IERX/02Fyb7eiDWvS/0U/2OHjYHp1oNSoijqn1yG7kIVR0n6a7k3HuhzaEHDn9BeMF97ghEHsskkGAZerGQHYutaZL8YTwT1/OtTChOw==",
        "X-MS-Exchange-Authentication-Results": "spf=pass (sender IP is 216.228.117.161)\n smtp.mailfrom=nvidia.com;\n dkim=none (message not signed)\n header.d=none;dmarc=pass action=none header.from=nvidia.com;",
        "Received-SPF": "Pass (protection.outlook.com: domain of nvidia.com designates\n 216.228.117.161 as permitted sender) receiver=protection.outlook.com;\n client-ip=216.228.117.161; helo=mail.nvidia.com; pr=C",
        "From": "Suanming Mou <suanmingm@nvidia.com>",
        "To": "Matan Azrad <matan@nvidia.com>, Viacheslav Ovsiienko\n <viacheslavo@nvidia.com>, Ray Kinsella <mdr@ashroe.eu>",
        "CC": "<dev@dpdk.org>, <rasland@nvidia.com>, <orika@nvidia.com>, Xiaoyu Min\n <jackmin@nvidia.com>",
        "Subject": "[PATCH v3 08/17] net/mlx5: add HW steering counter action",
        "Date": "Fri, 30 Sep 2022 15:53:06 +0300",
        "Message-ID": "<20220930125315.5079-9-suanmingm@nvidia.com>",
        "X-Mailer": "git-send-email 2.18.1",
        "In-Reply-To": "<20220930125315.5079-1-suanmingm@nvidia.com>",
        "References": "<20220923144334.27736-1-suanmingm@nvidia.com>\n <20220930125315.5079-1-suanmingm@nvidia.com>",
        "MIME-Version": "1.0",
        "Content-Type": "text/plain",
        "X-Originating-IP": "[10.126.230.35]",
        "X-ClientProxiedBy": "rnnvmail201.nvidia.com (10.129.68.8) To\n rnnvmail201.nvidia.com (10.129.68.8)",
        "X-EOPAttributedMessage": "0",
        "X-MS-PublicTrafficType": "Email",
        "X-MS-TrafficTypeDiagnostic": "DM6NAM11FT102:EE_|CY5PR12MB6180:EE_",
        "X-MS-Office365-Filtering-Correlation-Id": "7d53f3d2-65da-4393-a7d5-08daa2e2d5b5",
        "X-MS-Exchange-SenderADCheck": "1",
        "X-MS-Exchange-AntiSpam-Relay": "0",
        "X-Microsoft-Antispam": "BCL:0;",
        "X-Microsoft-Antispam-Message-Info": "\n dp1tLRNsRo4wW9QEJABGAWzyk8M0iOkhz3EvpwgmqN+g62e4iaKqUIUPCRQTE2gvp3Bxmb/BVMaofsMfn9Tism+eo2tfTiuT+r7fmAIdXEKTMDbXtlHcq57hDS5p8TAQ2jLyrNcE6qk3/PWvf7lfgZLcGtLYT6BWkexDWhyrMzL8Ae6yAoLP5elEcJ/fN+0eDUnVPUHjSRtyBaJxEK35BwV5os3nvNUKwurEXEQiPIr0LkIDnvuD8lRSGIFUu242XPr2csPiXXTm4+OwSqNBRe/0OGcVbhlFDdpkgBEokpbj9OY1kb96xQevddwY7+Yz6cWXxjbZmsSXaFcxrQsKd/61tJqpHoSDaG4ZzZgtsT/w16lpBBjZB/7xKTNV+uM3wPRuqdoKozyelkX/P5VjAIykpKDPQFN+bglIOngDDsMPyw6UOFRyFMJE0pnzOoKg7r6BUuQW/9zTm96r9trKztAwWCLasXV2hrmz0zPdEyGa2GEhKXJiSnwQu+zYpymlwN5hAYCKjiuBvViR41gGmJcyYSXez6xMsqDLoKZOK8ldHMS7uhW/Ow7hzAYvldTytl4ag7GUTjTf1V/M2ySQM2yzNtfTlTH3/vNZDByY+Rc1s9Zp2Qe8hJSGvtwcbEJtAbHicmdIKsYy0chONobuChLkZPsovz2y++L4WRnwfw5zKVC4cWlN553KrqneyZS1TyhY3h+RJEY0aW6Mnuex7nnZUEWeOIlritrKe1gpyCp+ki6EuHUnN07tuxy9uj3S8A0X79ld2ikeZqYgPCvuEpQyY4L7Z5eTfGLxRr6ztcM=",
        "X-Forefront-Antispam-Report": "CIP:216.228.117.161; CTRY:US; LANG:en; SCL:1;\n SRV:;\n IPV:NLI; SFV:NSPM; H:mail.nvidia.com; PTR:dc6edge2.nvidia.com; CAT:NONE;\n SFS:(13230022)(4636009)(396003)(39860400002)(136003)(376002)(346002)(451199015)(36840700001)(46966006)(40470700004)(36860700001)(86362001)(4326008)(6286002)(7636003)(356005)(40480700001)(83380400001)(70586007)(40460700003)(110136005)(82740400003)(8676002)(36756003)(54906003)(316002)(70206006)(426003)(41300700001)(336012)(55016003)(2906002)(5660300002)(30864003)(16526019)(1076003)(2616005)(47076005)(186003)(8936002)(478600001)(107886003)(6666004)(82310400005)(26005)(7696005)(579004)(559001)(309714004);\n DIR:OUT; SFP:1101;",
        "X-OriginatorOrg": "Nvidia.com",
        "X-MS-Exchange-CrossTenant-OriginalArrivalTime": "30 Sep 2022 12:53:55.6544 (UTC)",
        "X-MS-Exchange-CrossTenant-Network-Message-Id": "\n 7d53f3d2-65da-4393-a7d5-08daa2e2d5b5",
        "X-MS-Exchange-CrossTenant-Id": "43083d15-7273-40c1-b7db-39efd9ccc17a",
        "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp": "\n TenantId=43083d15-7273-40c1-b7db-39efd9ccc17a; Ip=[216.228.117.161];\n Helo=[mail.nvidia.com]",
        "X-MS-Exchange-CrossTenant-AuthSource": "\n DM6NAM11FT102.eop-nam11.prod.protection.outlook.com",
        "X-MS-Exchange-CrossTenant-AuthAs": "Anonymous",
        "X-MS-Exchange-CrossTenant-FromEntityHeader": "HybridOnPrem",
        "X-MS-Exchange-Transport-CrossTenantHeadersStamped": "CY5PR12MB6180",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org"
    },
    "content": "From: Xiaoyu Min <jackmin@nvidia.com>\n\nThis commit adds HW steering counter action support.\nPool mechanism is the basic data structure for the HW steering counter.\n\nThe HW steering's counter pool is based on the rte_ring of zero-copy\nvariation.\n\nThere are two global rte_rings:\n1. free_list:\n     Store the counters indexes, which are ready for use.\n2. wait_reset_list:\n     Store the counters indexes, which are just freed from the user and\n     need to query the hardware counter to get the reset value before\n     this counter can be reused again.\n\nThe counter pool also supports cache per HW steering's queues, which are\nalso based on rte_ring of zero-copy variation.\n\nThe cache can be configured in size, preload, threshold, and fetch size,\nthey are all exposed via device args.\n\nThe main operations of the counter pool are as follows:\n\n - Get one counter from the pool:\n   1. The user call _get_* API.\n   2. If the cache is enabled, dequeue one counter index from the local\n      cache:\n      2.A: if the dequeued one from the local cache is still in reset\n\tstatus (counter's query_gen_when_free is equal to pool's query\n\tgen):\n\tI. Flush all counters in local cache back to global\n\t   wait_reset_list.\n\tII. Fetch _fetch_sz_ counters into the cache from the global\n\t    free list.\n\tIII. Fetch one counter from the cache.\n   3. If the cache is empty, fetch _fetch_sz_ counters from the global\n      free list into the cache and fetch one counter from the cache.\n - Free one counter into the pool:\n   1. The user calls _put_* API.\n   2. Put the counter into the local cache.\n   3. If the local cache is full:\n      3.A: Write back all counters above _threshold_ into the global\n           wait_reset_list.\n      3.B: Also, write back this counter into the global wait_reset_list.\n\nWhen the local cache is disabled, _get_/_put_ cache directly from/into\nglobal list.\n\nSigned-off-by: Xiaoyu Min <jackmin@nvidia.com>\n---\n drivers/common/mlx5/mlx5_devx_cmds.c |  50 +++\n drivers/common/mlx5/mlx5_devx_cmds.h |  27 ++\n drivers/common/mlx5/mlx5_prm.h       |  62 ++-\n drivers/common/mlx5/version.map      |   1 +\n drivers/net/mlx5/meson.build         |   1 +\n drivers/net/mlx5/mlx5.c              |  14 +\n drivers/net/mlx5/mlx5.h              |  27 ++\n drivers/net/mlx5/mlx5_defs.h         |   2 +\n drivers/net/mlx5/mlx5_flow.c         |  27 +-\n drivers/net/mlx5/mlx5_flow.h         |   5 +\n drivers/net/mlx5/mlx5_flow_aso.c     | 261 ++++++++++++-\n drivers/net/mlx5/mlx5_flow_hw.c      | 340 +++++++++++++++-\n drivers/net/mlx5/mlx5_hws_cnt.c      | 528 +++++++++++++++++++++++++\n drivers/net/mlx5/mlx5_hws_cnt.h      | 558 +++++++++++++++++++++++++++\n 14 files changed, 1871 insertions(+), 32 deletions(-)\n create mode 100644 drivers/net/mlx5/mlx5_hws_cnt.c\n create mode 100644 drivers/net/mlx5/mlx5_hws_cnt.h",
    "diff": "diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c\nindex ac6891145d..eef7a98248 100644\n--- a/drivers/common/mlx5/mlx5_devx_cmds.c\n+++ b/drivers/common/mlx5/mlx5_devx_cmds.c\n@@ -176,6 +176,41 @@ mlx5_devx_cmd_register_write(void *ctx, uint16_t reg_id, uint32_t arg,\n \treturn 0;\n }\n \n+struct mlx5_devx_obj *\n+mlx5_devx_cmd_flow_counter_alloc_general(void *ctx,\n+\t\tstruct mlx5_devx_counter_attr *attr)\n+{\n+\tstruct mlx5_devx_obj *dcs = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*dcs),\n+\t\t\t\t\t\t0, SOCKET_ID_ANY);\n+\tuint32_t in[MLX5_ST_SZ_DW(alloc_flow_counter_in)]   = {0};\n+\tuint32_t out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {0};\n+\n+\tif (!dcs) {\n+\t\trte_errno = ENOMEM;\n+\t\treturn NULL;\n+\t}\n+\tMLX5_SET(alloc_flow_counter_in, in, opcode,\n+\t\t MLX5_CMD_OP_ALLOC_FLOW_COUNTER);\n+\tif (attr->bulk_log_max_alloc)\n+\t\tMLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk_log_size,\n+\t\t\t attr->flow_counter_bulk_log_size);\n+\telse\n+\t\tMLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk,\n+\t\t\t attr->bulk_n_128);\n+\tif (attr->pd_valid)\n+\t\tMLX5_SET(alloc_flow_counter_in, in, pd, attr->pd);\n+\tdcs->obj = mlx5_glue->devx_obj_create(ctx, in,\n+\t\t\t\t\t      sizeof(in), out, sizeof(out));\n+\tif (!dcs->obj) {\n+\t\tDRV_LOG(ERR, \"Can't allocate counters - error %d\", errno);\n+\t\trte_errno = errno;\n+\t\tmlx5_free(dcs);\n+\t\treturn NULL;\n+\t}\n+\tdcs->id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);\n+\treturn dcs;\n+}\n+\n /**\n  * Allocate flow counters via devx interface.\n  *\n@@ -967,6 +1002,16 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,\n \t\t\t\t\t general_obj_types) &\n \t\t\t      MLX5_GENERAL_OBJ_TYPES_CAP_CONN_TRACK_OFFLOAD);\n \tattr->rq_delay_drop = MLX5_GET(cmd_hca_cap, hcattr, rq_delay_drop);\n+\tattr->max_flow_counter_15_0 = MLX5_GET(cmd_hca_cap, hcattr,\n+\t\t\tmax_flow_counter_15_0);\n+\tattr->max_flow_counter_31_16 = MLX5_GET(cmd_hca_cap, hcattr,\n+\t\t\tmax_flow_counter_31_16);\n+\tattr->alloc_flow_counter_pd = MLX5_GET(cmd_hca_cap, hcattr,\n+\t\t\talloc_flow_counter_pd);\n+\tattr->flow_counter_access_aso = MLX5_GET(cmd_hca_cap, hcattr,\n+\t\t\tflow_counter_access_aso);\n+\tattr->flow_access_aso_opc_mod = MLX5_GET(cmd_hca_cap, hcattr,\n+\t\t\tflow_access_aso_opc_mod);\n \tif (attr->crypto) {\n \t\tattr->aes_xts = MLX5_GET(cmd_hca_cap, hcattr, aes_xts);\n \t\thcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,\n@@ -989,6 +1034,11 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,\n \t\t}\n \t\tattr->log_min_stride_wqe_sz = MLX5_GET(cmd_hca_cap_2, hcattr,\n \t\t\t\t\t\t       log_min_stride_wqe_sz);\n+\t\tattr->flow_counter_bulk_log_max_alloc = MLX5_GET(cmd_hca_cap_2,\n+\t\t\t\thcattr, flow_counter_bulk_log_max_alloc);\n+\t\tattr->flow_counter_bulk_log_granularity =\n+\t\t\tMLX5_GET(cmd_hca_cap_2, hcattr,\n+\t\t\t\t flow_counter_bulk_log_granularity);\n \t}\n \tif (attr->log_min_stride_wqe_sz == 0)\n \t\tattr->log_min_stride_wqe_sz = MLX5_MPRQ_LOG_MIN_STRIDE_WQE_SIZE;\ndiff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h\nindex d69dad613e..15b46f2acd 100644\n--- a/drivers/common/mlx5/mlx5_devx_cmds.h\n+++ b/drivers/common/mlx5/mlx5_devx_cmds.h\n@@ -15,6 +15,16 @@\n #define MLX5_DEVX_MAX_KLM_ENTRIES ((UINT16_MAX - \\\n \t\tMLX5_ST_SZ_DW(create_mkey_in) * 4) / (MLX5_ST_SZ_DW(klm) * 4))\n \n+struct mlx5_devx_counter_attr {\n+\tuint32_t pd_valid:1;\n+\tuint32_t pd:24;\n+\tuint32_t bulk_log_max_alloc:1;\n+\tunion {\n+\t\tuint8_t flow_counter_bulk_log_size;\n+\t\tuint8_t bulk_n_128;\n+\t};\n+};\n+\n struct mlx5_devx_mkey_attr {\n \tuint64_t addr;\n \tuint64_t size;\n@@ -263,6 +273,18 @@ struct mlx5_hca_attr {\n \tuint32_t set_reg_c:8;\n \tuint32_t nic_flow_table:1;\n \tuint32_t modify_outer_ip_ecn:1;\n+\tunion {\n+\t\tuint32_t max_flow_counter;\n+\t\tstruct {\n+\t\t\tuint16_t max_flow_counter_15_0;\n+\t\t\tuint16_t max_flow_counter_31_16;\n+\t\t};\n+\t};\n+\tuint32_t flow_counter_bulk_log_max_alloc:5;\n+\tuint32_t flow_counter_bulk_log_granularity:5;\n+\tuint32_t alloc_flow_counter_pd:1;\n+\tuint32_t flow_counter_access_aso:1;\n+\tuint32_t flow_access_aso_opc_mod:8;\n };\n \n /* LAG Context. */\n@@ -593,6 +615,11 @@ struct mlx5_devx_crypto_login_attr {\n \n /* mlx5_devx_cmds.c */\n \n+__rte_internal\n+struct mlx5_devx_obj *\n+mlx5_devx_cmd_flow_counter_alloc_general(void *ctx,\n+\t\t\t\tstruct mlx5_devx_counter_attr *attr);\n+\n __rte_internal\n struct mlx5_devx_obj *mlx5_devx_cmd_flow_counter_alloc(void *ctx,\n \t\t\t\t\t\t       uint32_t bulk_sz);\ndiff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h\nindex c82ec94465..8514ca8fc4 100644\n--- a/drivers/common/mlx5/mlx5_prm.h\n+++ b/drivers/common/mlx5/mlx5_prm.h\n@@ -1161,8 +1161,10 @@ struct mlx5_ifc_alloc_flow_counter_in_bits {\n \tu8 reserved_at_10[0x10];\n \tu8 reserved_at_20[0x10];\n \tu8 op_mod[0x10];\n-\tu8 flow_counter_id[0x20];\n-\tu8 reserved_at_40[0x18];\n+\tu8 reserved_at_40[0x8];\n+\tu8 pd[0x18];\n+\tu8 reserved_at_60[0x13];\n+\tu8 flow_counter_bulk_log_size[0x5];\n \tu8 flow_counter_bulk[0x8];\n };\n \n@@ -1382,7 +1384,13 @@ enum {\n #define MLX5_STEERING_LOGIC_FORMAT_CONNECTX_6DX 0x1\n \n struct mlx5_ifc_cmd_hca_cap_bits {\n-\tu8 reserved_at_0[0x20];\n+\tu8 access_other_hca_roce[0x1];\n+\tu8 alloc_flow_counter_pd[0x1];\n+\tu8 flow_counter_access_aso[0x1];\n+\tu8 reserved_at_3[0x5];\n+\tu8 flow_access_aso_opc_mod[0x8];\n+\tu8 reserved_at_10[0xf];\n+\tu8 vhca_resource_manager[0x1];\n \tu8 hca_cap_2[0x1];\n \tu8 reserved_at_21[0xf];\n \tu8 vhca_id[0x10];\n@@ -2058,8 +2066,52 @@ struct mlx5_ifc_cmd_hca_cap_2_bits {\n \tu8 log_conn_track_max_alloc[0x5];\n \tu8 reserved_at_d8[0x3];\n \tu8 log_max_conn_track_offload[0x5];\n-\tu8 reserved_at_e0[0x20]; /* End of DW7. */\n-\tu8 reserved_at_100[0x700];\n+\tu8 reserved_at_e0[0xc0];\n+\tu8 reserved_at_1a0[0xb];\n+\tu8 format_select_dw_8_6_ext[0x1];\n+\tu8 reserved_at_1ac[0x14];\n+\tu8 general_obj_types_127_64[0x40];\n+\tu8 reserved_at_200[0x53];\n+\tu8 flow_counter_bulk_log_max_alloc[0x5];\n+\tu8 reserved_at_258[0x3];\n+\tu8 flow_counter_bulk_log_granularity[0x5];\n+\tu8 reserved_at_260[0x20];\n+\tu8 format_select_dw_gtpu_dw_0[0x8];\n+\tu8 format_select_dw_gtpu_dw_1[0x8];\n+\tu8 format_select_dw_gtpu_dw_2[0x8];\n+\tu8 format_select_dw_gtpu_first_ext_dw_0[0x8];\n+\tu8 reserved_at_2a0[0x560];\n+};\n+\n+struct mlx5_ifc_wqe_based_flow_table_cap_bits {\n+\tu8 reserved_at_0[0x3];\n+\tu8 log_max_num_ste[0x5];\n+\tu8 reserved_at_8[0x3];\n+\tu8 log_max_num_stc[0x5];\n+\tu8 reserved_at_10[0x3];\n+\tu8 log_max_num_rtc[0x5];\n+\tu8 reserved_at_18[0x3];\n+\tu8 log_max_num_header_modify_pattern[0x5];\n+\tu8 reserved_at_20[0x3];\n+\tu8 stc_alloc_log_granularity[0x5];\n+\tu8 reserved_at_28[0x3];\n+\tu8 stc_alloc_log_max[0x5];\n+\tu8 reserved_at_30[0x3];\n+\tu8 ste_alloc_log_granularity[0x5];\n+\tu8 reserved_at_38[0x3];\n+\tu8 ste_alloc_log_max[0x5];\n+\tu8 reserved_at_40[0xb];\n+\tu8 rtc_reparse_mode[0x5];\n+\tu8 reserved_at_50[0x3];\n+\tu8 rtc_index_mode[0x5];\n+\tu8 reserved_at_58[0x3];\n+\tu8 rtc_log_depth_max[0x5];\n+\tu8 reserved_at_60[0x10];\n+\tu8 ste_format[0x10];\n+\tu8 stc_action_type[0x80];\n+\tu8 header_insert_type[0x10];\n+\tu8 header_remove_type[0x10];\n+\tu8 trivial_match_definer[0x20];\n };\n \n struct mlx5_ifc_esw_cap_bits {\ndiff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map\nindex 413dec14ab..4f72900519 100644\n--- a/drivers/common/mlx5/version.map\n+++ b/drivers/common/mlx5/version.map\n@@ -40,6 +40,7 @@ INTERNAL {\n \tmlx5_devx_cmd_create_virtq;\n \tmlx5_devx_cmd_destroy;\n \tmlx5_devx_cmd_flow_counter_alloc;\n+\tmlx5_devx_cmd_flow_counter_alloc_general;\n \tmlx5_devx_cmd_flow_counter_query;\n \tmlx5_devx_cmd_flow_dump;\n \tmlx5_devx_cmd_flow_single_dump;\ndiff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build\nindex 6a84d96380..f2d7bcaff6 100644\n--- a/drivers/net/mlx5/meson.build\n+++ b/drivers/net/mlx5/meson.build\n@@ -38,6 +38,7 @@ sources = files(\n         'mlx5_vlan.c',\n         'mlx5_utils.c',\n         'mlx5_devx.c',\n+\t'mlx5_hws_cnt.c',\n )\n \n if is_linux\ndiff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c\nindex cf5146d677..b6a66f12ee 100644\n--- a/drivers/net/mlx5/mlx5.c\n+++ b/drivers/net/mlx5/mlx5.c\n@@ -175,6 +175,12 @@\n /* Device parameter to create the fdb default rule in PMD */\n #define MLX5_FDB_DEFAULT_RULE_EN \"fdb_def_rule_en\"\n \n+/* HW steering counter configuration. */\n+#define MLX5_HWS_CNT_SERVICE_CORE \"service_core\"\n+\n+/* HW steering counter's query interval. */\n+#define MLX5_HWS_CNT_CYCLE_TIME \"svc_cycle_time\"\n+\n /* Shared memory between primary and secondary processes. */\n struct mlx5_shared_data *mlx5_shared_data;\n \n@@ -1245,6 +1251,10 @@ mlx5_dev_args_check_handler(const char *key, const char *val, void *opaque)\n \t\tconfig->allow_duplicate_pattern = !!tmp;\n \t} else if (strcmp(MLX5_FDB_DEFAULT_RULE_EN, key) == 0) {\n \t\tconfig->fdb_def_rule = !!tmp;\n+\t} else if (strcmp(MLX5_HWS_CNT_SERVICE_CORE, key) == 0) {\n+\t\tconfig->cnt_svc.service_core = tmp;\n+\t} else if (strcmp(MLX5_HWS_CNT_CYCLE_TIME, key) == 0) {\n+\t\tconfig->cnt_svc.cycle_time = tmp;\n \t}\n \treturn 0;\n }\n@@ -1281,6 +1291,8 @@ mlx5_shared_dev_ctx_args_config(struct mlx5_dev_ctx_shared *sh,\n \t\tMLX5_DECAP_EN,\n \t\tMLX5_ALLOW_DUPLICATE_PATTERN,\n \t\tMLX5_FDB_DEFAULT_RULE_EN,\n+\t\tMLX5_HWS_CNT_SERVICE_CORE,\n+\t\tMLX5_HWS_CNT_CYCLE_TIME,\n \t\tNULL,\n \t};\n \tint ret = 0;\n@@ -1293,6 +1305,8 @@ mlx5_shared_dev_ctx_args_config(struct mlx5_dev_ctx_shared *sh,\n \tconfig->decap_en = 1;\n \tconfig->allow_duplicate_pattern = 1;\n \tconfig->fdb_def_rule = 1;\n+\tconfig->cnt_svc.cycle_time = MLX5_CNT_SVC_CYCLE_TIME_DEFAULT;\n+\tconfig->cnt_svc.service_core = rte_get_main_lcore();\n \tif (mkvlist != NULL) {\n \t\t/* Process parameters. */\n \t\tret = mlx5_kvargs_process(mkvlist, params,\ndiff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h\nindex 686969719a..4859f5a509 100644\n--- a/drivers/net/mlx5/mlx5.h\n+++ b/drivers/net/mlx5/mlx5.h\n@@ -308,6 +308,10 @@ struct mlx5_sh_config {\n \tuint32_t hw_fcs_strip:1; /* FCS stripping is supported. */\n \tuint32_t allow_duplicate_pattern:1;\n \tuint32_t lro_allowed:1; /* Whether LRO is allowed. */\n+\tstruct {\n+\t\tuint16_t service_core;\n+\t\tuint32_t cycle_time; /* query cycle time in milli-second. */\n+\t} cnt_svc; /* configure for HW steering's counter's service. */\n \t/* Allow/Prevent the duplicate rules pattern. */\n \tuint32_t fdb_def_rule:1; /* Create FDB default jump rule */\n };\n@@ -1224,6 +1228,22 @@ struct mlx5_flex_item {\n \tstruct mlx5_flex_pattern_field map[MLX5_FLEX_ITEM_MAPPING_NUM];\n };\n \n+#define HWS_CNT_ASO_SQ_NUM 4\n+\n+struct mlx5_hws_aso_mng {\n+\tuint16_t sq_num;\n+\tstruct mlx5_aso_sq sqs[HWS_CNT_ASO_SQ_NUM];\n+};\n+\n+struct mlx5_hws_cnt_svc_mng {\n+\tuint32_t refcnt;\n+\tuint32_t service_core;\n+\tuint32_t query_interval;\n+\tpthread_t service_thread;\n+\tuint8_t svc_running;\n+\tstruct mlx5_hws_aso_mng aso_mng __rte_cache_aligned;\n+};\n+\n /*\n  * Shared Infiniband device context for Master/Representors\n  * which belong to same IB device with multiple IB ports.\n@@ -1323,6 +1343,7 @@ struct mlx5_dev_ctx_shared {\n \tpthread_mutex_t lwm_config_lock;\n \tuint32_t host_shaper_rate:8;\n \tuint32_t lwm_triggered:1;\n+\tstruct mlx5_hws_cnt_svc_mng *cnt_svc;\n \tstruct mlx5_dev_shared_port port[]; /* per device port data array. */\n };\n \n@@ -1607,6 +1628,7 @@ struct mlx5_priv {\n \t/* Flex items have been created on the port. */\n \tuint32_t flex_item_map; /* Map of allocated flex item elements. */\n \tuint32_t nb_queue; /* HW steering queue number. */\n+\tstruct mlx5_hws_cnt_pool *hws_cpool; /* HW steering's counter pool. */\n #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)\n \t/* Item template list. */\n \tLIST_HEAD(flow_hw_itt, rte_flow_pattern_template) flow_hw_itt;\n@@ -2037,6 +2059,11 @@ mlx5_get_supported_sw_parsing_offloads(const struct mlx5_hca_attr *attr);\n uint32_t\n mlx5_get_supported_tunneling_offloads(const struct mlx5_hca_attr *attr);\n \n+int mlx5_aso_cnt_queue_init(struct mlx5_dev_ctx_shared *sh);\n+void mlx5_aso_cnt_queue_uninit(struct mlx5_dev_ctx_shared *sh);\n+int mlx5_aso_cnt_query(struct mlx5_dev_ctx_shared *sh,\n+\t\tstruct mlx5_hws_cnt_pool *cpool);\n+\n /* mlx5_flow_flex.c */\n \n struct rte_flow_item_flex_handle *\ndiff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h\nindex 585afb0a98..d064abfef3 100644\n--- a/drivers/net/mlx5/mlx5_defs.h\n+++ b/drivers/net/mlx5/mlx5_defs.h\n@@ -188,4 +188,6 @@\n #define static_assert _Static_assert\n #endif\n \n+#define MLX5_CNT_SVC_CYCLE_TIME_DEFAULT 500\n+\n #endif /* RTE_PMD_MLX5_DEFS_H_ */\ndiff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c\nindex fb3be940e5..658cc69750 100644\n--- a/drivers/net/mlx5/mlx5_flow.c\n+++ b/drivers/net/mlx5/mlx5_flow.c\n@@ -7832,24 +7832,33 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,\n  */\n static int\n flow_drv_query(struct rte_eth_dev *dev,\n-\t       uint32_t flow_idx,\n+\t       struct rte_flow *eflow,\n \t       const struct rte_flow_action *actions,\n \t       void *data,\n \t       struct rte_flow_error *error)\n {\n \tstruct mlx5_priv *priv = dev->data->dev_private;\n \tconst struct mlx5_flow_driver_ops *fops;\n-\tstruct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],\n-\t\t\t\t\t       flow_idx);\n-\tenum mlx5_flow_drv_type ftype;\n+\tstruct rte_flow *flow = NULL;\n+\tenum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN;\n \n+\tif (priv->sh->config.dv_flow_en == 2) {\n+#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)\n+\t\tflow = eflow;\n+\t\tftype = MLX5_FLOW_TYPE_HW;\n+#endif\n+\t} else {\n+\t\tflow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],\n+\t\t\t\t(uintptr_t)(void *)eflow);\n+\t}\n \tif (!flow) {\n \t\treturn rte_flow_error_set(error, ENOENT,\n \t\t\t  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,\n \t\t\t  NULL,\n \t\t\t  \"invalid flow handle\");\n \t}\n-\tftype = flow->drv_type;\n+\tif (ftype == MLX5_FLOW_TYPE_MIN)\n+\t\tftype = flow->drv_type;\n \tMLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);\n \tfops = flow_get_drv_ops(ftype);\n \n@@ -7870,14 +7879,8 @@ mlx5_flow_query(struct rte_eth_dev *dev,\n \t\tstruct rte_flow_error *error)\n {\n \tint ret;\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n \n-\tif (priv->sh->config.dv_flow_en == 2)\n-\t\treturn rte_flow_error_set(error, ENOTSUP,\n-\t\t\t  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,\n-\t\t\t  NULL,\n-\t\t\t  \"Flow non-Q query not supported\");\n-\tret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,\n+\tret = flow_drv_query(dev, flow, actions, data,\n \t\t\t     error);\n \tif (ret < 0)\n \t\treturn ret;\ndiff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h\nindex 3bde95c927..8f1b66eaac 100644\n--- a/drivers/net/mlx5/mlx5_flow.h\n+++ b/drivers/net/mlx5/mlx5_flow.h\n@@ -1103,6 +1103,7 @@ struct rte_flow_hw {\n \t};\n \tstruct rte_flow_template_table *table; /* The table flow allcated from. */\n \tstruct mlx5dr_rule rule; /* HWS layer data struct. */\n+\tuint32_t cnt_id;\n } __rte_packed;\n \n /* rte flow action translate to DR action struct. */\n@@ -1146,6 +1147,9 @@ struct mlx5_action_construct_data {\n \t\t\tuint32_t level; /* RSS level. */\n \t\t\tuint32_t idx; /* Shared action index. */\n \t\t} shared_rss;\n+\t\tstruct {\n+\t\t\tuint32_t id;\n+\t\t} shared_counter;\n \t};\n };\n \n@@ -1224,6 +1228,7 @@ struct mlx5_hw_actions {\n \tuint16_t encap_decap_pos; /* Encap/Decap action position. */\n \tuint32_t acts_num:4; /* Total action number. */\n \tuint32_t mark:1; /* Indicate the mark action. */\n+\tuint32_t cnt_id; /* Counter id. */\n \t/* Translated DR action array from action template. */\n \tstruct mlx5dr_rule_action rule_acts[MLX5_HW_MAX_ACTS];\n };\ndiff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c\nindex 60d0280367..ed9272e583 100644\n--- a/drivers/net/mlx5/mlx5_flow_aso.c\n+++ b/drivers/net/mlx5/mlx5_flow_aso.c\n@@ -12,6 +12,9 @@\n \n #include \"mlx5.h\"\n #include \"mlx5_flow.h\"\n+#include \"mlx5_hws_cnt.h\"\n+\n+#define MLX5_ASO_CNT_QUEUE_LOG_DESC 14\n \n /**\n  * Free MR resources.\n@@ -79,6 +82,33 @@ mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq)\n \tmemset(sq, 0, sizeof(*sq));\n }\n \n+/**\n+ * Initialize Send Queue used for ASO access counter.\n+ *\n+ * @param[in] sq\n+ *   ASO SQ to initialize.\n+ */\n+static void\n+mlx5_aso_cnt_init_sq(struct mlx5_aso_sq *sq)\n+{\n+\tvolatile struct mlx5_aso_wqe *restrict wqe;\n+\tint i;\n+\tint size = 1 << sq->log_desc_n;\n+\n+\t/* All the next fields state should stay constant. */\n+\tfor (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {\n+\t\twqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |\n+\t\t\t\t\t\t\t  (sizeof(*wqe) >> 4));\n+\t\twqe->aso_cseg.operand_masks = rte_cpu_to_be_32\n+\t\t\t(0u |\n+\t\t\t (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |\n+\t\t\t (ASO_OP_ALWAYS_FALSE << ASO_CSEG_COND_1_OPER_OFFSET) |\n+\t\t\t (ASO_OP_ALWAYS_FALSE << ASO_CSEG_COND_0_OPER_OFFSET) |\n+\t\t\t (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));\n+\t\twqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX);\n+\t}\n+}\n+\n /**\n  * Initialize Send Queue used for ASO access.\n  *\n@@ -191,7 +221,7 @@ mlx5_aso_ct_init_sq(struct mlx5_aso_sq *sq)\n  */\n static int\n mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq,\n-\t\t   void *uar)\n+\t\t   void *uar, uint16_t log_desc_n)\n {\n \tstruct mlx5_devx_cq_attr cq_attr = {\n \t\t.uar_page_id = mlx5_os_get_devx_uar_page_id(uar),\n@@ -212,12 +242,12 @@ mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq,\n \tint ret;\n \n \tif (mlx5_devx_cq_create(cdev->ctx, &sq->cq.cq_obj,\n-\t\t\t\tMLX5_ASO_QUEUE_LOG_DESC, &cq_attr,\n+\t\t\t\tlog_desc_n, &cq_attr,\n \t\t\t\tSOCKET_ID_ANY))\n \t\tgoto error;\n \tsq->cq.cq_ci = 0;\n-\tsq->cq.log_desc_n = MLX5_ASO_QUEUE_LOG_DESC;\n-\tsq->log_desc_n = MLX5_ASO_QUEUE_LOG_DESC;\n+\tsq->cq.log_desc_n = log_desc_n;\n+\tsq->log_desc_n = log_desc_n;\n \tsq_attr.cqn = sq->cq.cq_obj.cq->id;\n \t/* for mlx5_aso_wqe that is twice the size of mlx5_wqe */\n \tlog_wqbb_n = sq->log_desc_n + 1;\n@@ -269,7 +299,8 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,\n \t\t\t\t    sq_desc_n, &sh->aso_age_mng->aso_sq.mr))\n \t\t\treturn -1;\n \t\tif (mlx5_aso_sq_create(cdev, &sh->aso_age_mng->aso_sq,\n-\t\t\t\t       sh->tx_uar.obj)) {\n+\t\t\t\t       sh->tx_uar.obj,\n+\t\t\t\t       MLX5_ASO_QUEUE_LOG_DESC)) {\n \t\t\tmlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr);\n \t\t\treturn -1;\n \t\t}\n@@ -277,7 +308,7 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,\n \t\tbreak;\n \tcase ASO_OPC_MOD_POLICER:\n \t\tif (mlx5_aso_sq_create(cdev, &sh->mtrmng->pools_mng.sq,\n-\t\t\t\t       sh->tx_uar.obj))\n+\t\t\t\t       sh->tx_uar.obj, MLX5_ASO_QUEUE_LOG_DESC))\n \t\t\treturn -1;\n \t\tmlx5_aso_mtr_init_sq(&sh->mtrmng->pools_mng.sq);\n \t\tbreak;\n@@ -287,7 +318,7 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,\n \t\t\t\t    &sh->ct_mng->aso_sq.mr))\n \t\t\treturn -1;\n \t\tif (mlx5_aso_sq_create(cdev, &sh->ct_mng->aso_sq,\n-\t\t\t\t       sh->tx_uar.obj)) {\n+\t\t\t\t       sh->tx_uar.obj, MLX5_ASO_QUEUE_LOG_DESC)) {\n \t\t\tmlx5_aso_dereg_mr(cdev, &sh->ct_mng->aso_sq.mr);\n \t\t\treturn -1;\n \t\t}\n@@ -1403,3 +1434,219 @@ mlx5_aso_ct_available(struct mlx5_dev_ctx_shared *sh,\n \trte_errno = EBUSY;\n \treturn -rte_errno;\n }\n+\n+int\n+mlx5_aso_cnt_queue_init(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tstruct mlx5_hws_aso_mng *aso_mng = NULL;\n+\tuint8_t idx;\n+\tstruct mlx5_aso_sq *sq;\n+\n+\tMLX5_ASSERT(sh);\n+\tMLX5_ASSERT(sh->cnt_svc);\n+\taso_mng = &sh->cnt_svc->aso_mng;\n+\taso_mng->sq_num = HWS_CNT_ASO_SQ_NUM;\n+\tfor (idx = 0; idx < HWS_CNT_ASO_SQ_NUM; idx++) {\n+\t\tsq = &aso_mng->sqs[idx];\n+\t\tif (mlx5_aso_sq_create(sh->cdev, sq, sh->tx_uar.obj,\n+\t\t\t\t\tMLX5_ASO_CNT_QUEUE_LOG_DESC))\n+\t\t\tgoto error;\n+\t\tmlx5_aso_cnt_init_sq(sq);\n+\t}\n+\treturn 0;\n+error:\n+\tmlx5_aso_cnt_queue_uninit(sh);\n+\treturn -1;\n+}\n+\n+void\n+mlx5_aso_cnt_queue_uninit(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tuint16_t idx;\n+\n+\tfor (idx = 0; idx < sh->cnt_svc->aso_mng.sq_num; idx++)\n+\t\tmlx5_aso_destroy_sq(&sh->cnt_svc->aso_mng.sqs[idx]);\n+\tsh->cnt_svc->aso_mng.sq_num = 0;\n+}\n+\n+static uint16_t\n+mlx5_aso_cnt_sq_enqueue_burst(struct mlx5_hws_cnt_pool *cpool,\n+\t\tstruct mlx5_dev_ctx_shared *sh,\n+\t\tstruct mlx5_aso_sq *sq, uint32_t n,\n+\t\tuint32_t offset, uint32_t dcs_id_base)\n+{\n+\tvolatile struct mlx5_aso_wqe *wqe;\n+\tuint16_t size = 1 << sq->log_desc_n;\n+\tuint16_t mask = size - 1;\n+\tuint16_t max;\n+\tuint32_t upper_offset = offset;\n+\tuint64_t addr;\n+\tuint32_t ctrl_gen_id = 0;\n+\tuint8_t opcmod = sh->cdev->config.hca_attr.flow_access_aso_opc_mod;\n+\trte_be32_t lkey = rte_cpu_to_be_32(cpool->raw_mng->mr.lkey);\n+\tuint16_t aso_n = (uint16_t)(RTE_ALIGN_CEIL(n, 4) / 4);\n+\tuint32_t ccntid;\n+\n+\tmax = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), aso_n);\n+\tif (unlikely(!max))\n+\t\treturn 0;\n+\tupper_offset += (max * 4);\n+\t/* Because only one burst at one time, we can use the same elt. */\n+\tsq->elts[0].burst_size = max;\n+\tctrl_gen_id = dcs_id_base;\n+\tctrl_gen_id /= 4;\n+\tdo {\n+\t\tccntid = upper_offset - max * 4;\n+\t\twqe = &sq->sq_obj.aso_wqes[sq->head & mask];\n+\t\trte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);\n+\t\twqe->general_cseg.misc = rte_cpu_to_be_32(ctrl_gen_id);\n+\t\twqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<\n+\t\t\t\t\t\t\t MLX5_COMP_MODE_OFFSET);\n+\t\twqe->general_cseg.opcode = rte_cpu_to_be_32\n+\t\t\t\t\t\t(MLX5_OPCODE_ACCESS_ASO |\n+\t\t\t\t\t\t (opcmod <<\n+\t\t\t\t\t\t  WQE_CSEG_OPC_MOD_OFFSET) |\n+\t\t\t\t\t\t (sq->pi <<\n+\t\t\t\t\t\t  WQE_CSEG_WQE_INDEX_OFFSET));\n+\t\taddr = (uint64_t)RTE_PTR_ADD(cpool->raw_mng->raw,\n+\t\t\t\tccntid * sizeof(struct flow_counter_stats));\n+\t\twqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));\n+\t\twqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);\n+\t\twqe->aso_cseg.lkey = lkey;\n+\t\tsq->pi += 2; /* Each WQE contains 2 WQEBB's. */\n+\t\tsq->head++;\n+\t\tsq->next++;\n+\t\tctrl_gen_id++;\n+\t\tmax--;\n+\t} while (max);\n+\twqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<\n+\t\t\t\t\t\t\t MLX5_COMP_MODE_OFFSET);\n+\tmlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,\n+\t\t\t   sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],\n+\t\t\t   !sh->tx_uar.dbnc);\n+\treturn sq->elts[0].burst_size;\n+}\n+\n+static uint16_t\n+mlx5_aso_cnt_completion_handle(struct mlx5_aso_sq *sq)\n+{\n+\tstruct mlx5_aso_cq *cq = &sq->cq;\n+\tvolatile struct mlx5_cqe *restrict cqe;\n+\tconst unsigned int cq_size = 1 << cq->log_desc_n;\n+\tconst unsigned int mask = cq_size - 1;\n+\tuint32_t idx;\n+\tuint32_t next_idx = cq->cq_ci & mask;\n+\tconst uint16_t max = (uint16_t)(sq->head - sq->tail);\n+\tuint16_t i = 0;\n+\tint ret;\n+\tif (unlikely(!max))\n+\t\treturn 0;\n+\tidx = next_idx;\n+\tnext_idx = (cq->cq_ci + 1) & mask;\n+\trte_prefetch0(&cq->cq_obj.cqes[next_idx]);\n+\tcqe = &cq->cq_obj.cqes[idx];\n+\tret = check_cqe(cqe, cq_size, cq->cq_ci);\n+\t/*\n+\t * Be sure owner read is done before any other cookie field or\n+\t * opaque field.\n+\t */\n+\trte_io_rmb();\n+\tif (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {\n+\t\tif (likely(ret == MLX5_CQE_STATUS_HW_OWN))\n+\t\t\treturn 0; /* return immediately. */\n+\t\tmlx5_aso_cqe_err_handle(sq);\n+\t}\n+\ti += sq->elts[0].burst_size;\n+\tsq->elts[0].burst_size = 0;\n+\tcq->cq_ci++;\n+\tif (likely(i)) {\n+\t\tsq->tail += i;\n+\t\trte_io_wmb();\n+\t\tcq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);\n+\t}\n+\treturn i;\n+}\n+\n+static uint16_t\n+mlx5_aso_cnt_query_one_dcs(struct mlx5_dev_ctx_shared *sh,\n+\t\t\t   struct mlx5_hws_cnt_pool *cpool,\n+\t\t\t   uint8_t dcs_idx, uint32_t num)\n+{\n+\tuint32_t dcs_id = cpool->dcs_mng.dcs[dcs_idx].obj->id;\n+\tuint64_t cnt_num = cpool->dcs_mng.dcs[dcs_idx].batch_sz;\n+\tuint64_t left;\n+\tuint32_t iidx = cpool->dcs_mng.dcs[dcs_idx].iidx;\n+\tuint32_t offset;\n+\tuint16_t mask;\n+\tuint16_t sq_idx;\n+\tuint64_t burst_sz = (uint64_t)(1 << MLX5_ASO_CNT_QUEUE_LOG_DESC) * 4 *\n+\t\tsh->cnt_svc->aso_mng.sq_num;\n+\tuint64_t qburst_sz = burst_sz / sh->cnt_svc->aso_mng.sq_num;\n+\tuint64_t n;\n+\tstruct mlx5_aso_sq *sq;\n+\n+\tcnt_num = RTE_MIN(num, cnt_num);\n+\tleft = cnt_num;\n+\twhile (left) {\n+\t\tmask = 0;\n+\t\tfor (sq_idx = 0; sq_idx < sh->cnt_svc->aso_mng.sq_num;\n+\t\t\t\tsq_idx++) {\n+\t\t\tif (left == 0) {\n+\t\t\t\tmask |= (1 << sq_idx);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tn = RTE_MIN(left, qburst_sz);\n+\t\t\toffset = cnt_num - left;\n+\t\t\toffset += iidx;\n+\t\t\tmlx5_aso_cnt_sq_enqueue_burst(cpool, sh,\n+\t\t\t\t\t&sh->cnt_svc->aso_mng.sqs[sq_idx], n,\n+\t\t\t\t\toffset, dcs_id);\n+\t\t\tleft -= n;\n+\t\t}\n+\t\tdo {\n+\t\t\tfor (sq_idx = 0; sq_idx < sh->cnt_svc->aso_mng.sq_num;\n+\t\t\t\t\tsq_idx++) {\n+\t\t\t\tsq = &sh->cnt_svc->aso_mng.sqs[sq_idx];\n+\t\t\t\tif (mlx5_aso_cnt_completion_handle(sq))\n+\t\t\t\t\tmask |= (1 << sq_idx);\n+\t\t\t}\n+\t\t} while (mask < ((1 << sh->cnt_svc->aso_mng.sq_num) - 1));\n+\t}\n+\treturn cnt_num;\n+}\n+\n+/*\n+ * Query FW counter via ASO WQE.\n+ *\n+ * ASO query counter use _sync_ mode, means:\n+ * 1. each SQ issue one burst with several WQEs\n+ * 2. ask for CQE at last WQE\n+ * 3. busy poll CQ of each SQ's\n+ * 4. If all SQ's CQE are received then goto step 1, issue next burst\n+ *\n+ * @param[in] sh\n+ *   Pointer to shared device.\n+ * @param[in] cpool\n+ *   Pointer to counter pool.\n+ *\n+ * @return\n+ *   0 on success, -1 on failure.\n+ */\n+int\n+mlx5_aso_cnt_query(struct mlx5_dev_ctx_shared *sh,\n+\t\t   struct mlx5_hws_cnt_pool *cpool)\n+{\n+\tuint32_t idx;\n+\tuint32_t num;\n+\tuint32_t cnt_num = mlx5_hws_cnt_pool_get_size(cpool) -\n+\t\trte_ring_count(cpool->free_list);\n+\n+\tfor (idx = 0; idx < cpool->dcs_mng.batch_total; idx++) {\n+\t\tnum = RTE_MIN(cnt_num, cpool->dcs_mng.dcs[idx].batch_sz);\n+\t\tmlx5_aso_cnt_query_one_dcs(sh, cpool, idx, num);\n+\t\tcnt_num -= num;\n+\t\tif (cnt_num == 0)\n+\t\t\tbreak;\n+\t}\n+\treturn 0;\n+}\ndiff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c\nindex c2e16bc56d..507abb54e4 100644\n--- a/drivers/net/mlx5/mlx5_flow_hw.c\n+++ b/drivers/net/mlx5/mlx5_flow_hw.c\n@@ -10,6 +10,7 @@\n #include \"mlx5_rx.h\"\n \n #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)\n+#include \"mlx5_hws_cnt.h\"\n \n /* The maximum actions support in the flow. */\n #define MLX5_HW_MAX_ACTS 16\n@@ -353,6 +354,10 @@ __flow_hw_action_template_destroy(struct rte_eth_dev *dev,\n \t\t\tmlx5dr_action_destroy(acts->mhdr->action);\n \t\tmlx5_free(acts->mhdr);\n \t}\n+\tif (mlx5_hws_cnt_id_valid(acts->cnt_id)) {\n+\t\tmlx5_hws_cnt_shared_put(priv->hws_cpool, &acts->cnt_id);\n+\t\tacts->cnt_id = 0;\n+\t}\n }\n \n /**\n@@ -532,6 +537,44 @@ __flow_hw_act_data_shared_rss_append(struct mlx5_priv *priv,\n \treturn 0;\n }\n \n+/**\n+ * Append shared counter action to the dynamic action list.\n+ *\n+ * @param[in] priv\n+ *   Pointer to the port private data structure.\n+ * @param[in] acts\n+ *   Pointer to the template HW steering DR actions.\n+ * @param[in] type\n+ *   Action type.\n+ * @param[in] action_src\n+ *   Offset of source rte flow action.\n+ * @param[in] action_dst\n+ *   Offset of destination DR action.\n+ * @param[in] cnt_id\n+ *   Shared counter id.\n+ *\n+ * @return\n+ *    0 on success, negative value otherwise and rte_errno is set.\n+ */\n+static __rte_always_inline int\n+__flow_hw_act_data_shared_cnt_append(struct mlx5_priv *priv,\n+\t\t\t\t     struct mlx5_hw_actions *acts,\n+\t\t\t\t     enum rte_flow_action_type type,\n+\t\t\t\t     uint16_t action_src,\n+\t\t\t\t     uint16_t action_dst,\n+\t\t\t\t     cnt_id_t cnt_id)\n+{\tstruct mlx5_action_construct_data *act_data;\n+\n+\tact_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);\n+\tif (!act_data)\n+\t\treturn -1;\n+\tact_data->type = type;\n+\tact_data->shared_counter.id = cnt_id;\n+\tLIST_INSERT_HEAD(&acts->act_list, act_data, next);\n+\treturn 0;\n+}\n+\n+\n /**\n  * Translate shared indirect action.\n  *\n@@ -573,6 +616,13 @@ flow_hw_shared_action_translate(struct rte_eth_dev *dev,\n \t\t    action_src, action_dst, idx, shared_rss))\n \t\t\treturn -1;\n \t\tbreak;\n+\tcase MLX5_INDIRECT_ACTION_TYPE_COUNT:\n+\t\tif (__flow_hw_act_data_shared_cnt_append(priv, acts,\n+\t\t\t(enum rte_flow_action_type)\n+\t\t\tMLX5_RTE_FLOW_ACTION_TYPE_COUNT,\n+\t\t\taction_src, action_dst, act_idx))\n+\t\t\treturn -1;\n+\t\tbreak;\n \tdefault:\n \t\tDRV_LOG(WARNING, \"Unsupported shared action type:%d\", type);\n \t\tbreak;\n@@ -946,6 +996,30 @@ flow_hw_meter_compile(struct rte_eth_dev *dev,\n \t}\n \treturn 0;\n }\n+\n+static __rte_always_inline int\n+flow_hw_cnt_compile(struct rte_eth_dev *dev, uint32_t  start_pos,\n+\t\t      struct mlx5_hw_actions *acts)\n+{\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tuint32_t pos = start_pos;\n+\tcnt_id_t cnt_id;\n+\tint ret;\n+\n+\tret = mlx5_hws_cnt_shared_get(priv->hws_cpool, &cnt_id);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\tret = mlx5_hws_cnt_pool_get_action_offset\n+\t\t\t\t(priv->hws_cpool,\n+\t\t\t\t cnt_id,\n+\t\t\t\t &acts->rule_acts[pos].action,\n+\t\t\t\t &acts->rule_acts[pos].counter.offset);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\tacts->cnt_id = cnt_id;\n+\treturn 0;\n+}\n+\n /**\n  * Translate rte_flow actions to DR action.\n  *\n@@ -1189,6 +1263,20 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,\n \t\t\t\tgoto err;\n \t\t\ti++;\n \t\t\tbreak;\n+\t\tcase RTE_FLOW_ACTION_TYPE_COUNT:\n+\t\t\tif (masks->conf &&\n+\t\t\t    ((const struct rte_flow_action_count *)\n+\t\t\t     masks->conf)->id) {\n+\t\t\t\terr = flow_hw_cnt_compile(dev, i, acts);\n+\t\t\t\tif (err)\n+\t\t\t\t\tgoto err;\n+\t\t\t} else if (__flow_hw_act_data_general_append\n+\t\t\t\t\t(priv, acts, actions->type,\n+\t\t\t\t\t actions - action_start, i)) {\n+\t\t\t\tgoto err;\n+\t\t\t}\n+\t\t\ti++;\n+\t\t\tbreak;\n \t\tcase RTE_FLOW_ACTION_TYPE_END:\n \t\t\tactions_end = true;\n \t\t\tbreak;\n@@ -1377,6 +1465,13 @@ flow_hw_shared_action_construct(struct rte_eth_dev *dev,\n \t\t\t\t(dev, &act_data, item_flags, rule_act))\n \t\t\treturn -1;\n \t\tbreak;\n+\tcase MLX5_INDIRECT_ACTION_TYPE_COUNT:\n+\t\tif (mlx5_hws_cnt_pool_get_action_offset(priv->hws_cpool,\n+\t\t\t\tact_idx,\n+\t\t\t\t&rule_act->action,\n+\t\t\t\t&rule_act->counter.offset))\n+\t\t\treturn -1;\n+\t\tbreak;\n \tdefault:\n \t\tDRV_LOG(WARNING, \"Unsupported shared action type:%d\", type);\n \t\tbreak;\n@@ -1520,7 +1615,8 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,\n \t\t\t  const uint8_t it_idx,\n \t\t\t  const struct rte_flow_action actions[],\n \t\t\t  struct mlx5dr_rule_action *rule_acts,\n-\t\t\t  uint32_t *acts_num)\n+\t\t\t  uint32_t *acts_num,\n+\t\t\t  uint32_t queue)\n {\n \tstruct mlx5_priv *priv = dev->data->dev_private;\n \tstruct rte_flow_template_table *table = job->flow->table;\n@@ -1574,6 +1670,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,\n \t\tuint64_t item_flags;\n \t\tstruct mlx5_hw_jump_action *jump;\n \t\tstruct mlx5_hrxq *hrxq;\n+\t\tcnt_id_t cnt_id;\n \n \t\taction = &actions[act_data->action_src];\n \t\tMLX5_ASSERT(action->type == RTE_FLOW_ACTION_TYPE_INDIRECT ||\n@@ -1681,6 +1778,32 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,\n \t\t\tif (mlx5_aso_mtr_wait(priv->sh, mtr))\n \t\t\t\treturn -1;\n \t\t\tbreak;\n+\t\tcase RTE_FLOW_ACTION_TYPE_COUNT:\n+\t\t\tret = mlx5_hws_cnt_pool_get(priv->hws_cpool, &queue,\n+\t\t\t\t\t&cnt_id);\n+\t\t\tif (ret != 0)\n+\t\t\t\treturn ret;\n+\t\t\tret = mlx5_hws_cnt_pool_get_action_offset\n+\t\t\t\t(priv->hws_cpool,\n+\t\t\t\t cnt_id,\n+\t\t\t\t &rule_acts[act_data->action_dst].action,\n+\t\t\t\t &rule_acts[act_data->action_dst].counter.offset\n+\t\t\t\t );\n+\t\t\tif (ret != 0)\n+\t\t\t\treturn ret;\n+\t\t\tjob->flow->cnt_id = cnt_id;\n+\t\t\tbreak;\n+\t\tcase MLX5_RTE_FLOW_ACTION_TYPE_COUNT:\n+\t\t\tret = mlx5_hws_cnt_pool_get_action_offset\n+\t\t\t\t(priv->hws_cpool,\n+\t\t\t\t act_data->shared_counter.id,\n+\t\t\t\t &rule_acts[act_data->action_dst].action,\n+\t\t\t\t &rule_acts[act_data->action_dst].counter.offset\n+\t\t\t\t );\n+\t\t\tif (ret != 0)\n+\t\t\t\treturn ret;\n+\t\t\tjob->flow->cnt_id = act_data->shared_counter.id;\n+\t\t\tbreak;\n \t\tdefault:\n \t\t\tbreak;\n \t\t}\n@@ -1690,6 +1813,8 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,\n \t\t\t\tjob->flow->idx - 1;\n \t\trule_acts[hw_acts->encap_decap_pos].reformat.data = buf;\n \t}\n+\tif (mlx5_hws_cnt_id_valid(hw_acts->cnt_id))\n+\t\tjob->flow->cnt_id = hw_acts->cnt_id;\n \treturn 0;\n }\n \n@@ -1825,7 +1950,7 @@ flow_hw_async_flow_create(struct rte_eth_dev *dev,\n \t * user's input, in order to save the cost.\n \t */\n \tif (flow_hw_actions_construct(dev, job, hw_acts, pattern_template_index,\n-\t\t\t\t  actions, rule_acts, &acts_num)) {\n+\t\t\t\t  actions, rule_acts, &acts_num, queue)) {\n \t\trte_errno = EINVAL;\n \t\tgoto free;\n \t}\n@@ -1955,6 +2080,13 @@ flow_hw_pull(struct rte_eth_dev *dev,\n \t\t\t\tflow_hw_jump_release(dev, job->flow->jump);\n \t\t\telse if (job->flow->fate_type == MLX5_FLOW_FATE_QUEUE)\n \t\t\t\tmlx5_hrxq_obj_release(dev, job->flow->hrxq);\n+\t\t\tif (mlx5_hws_cnt_id_valid(job->flow->cnt_id) &&\n+\t\t\t    mlx5_hws_cnt_is_shared\n+\t\t\t\t(priv->hws_cpool, job->flow->cnt_id) == false) {\n+\t\t\t\tmlx5_hws_cnt_pool_put(priv->hws_cpool, &queue,\n+\t\t\t\t\t\t&job->flow->cnt_id);\n+\t\t\t\tjob->flow->cnt_id = 0;\n+\t\t\t}\n \t\t\tmlx5_ipool_free(job->flow->table->flow, job->flow->idx);\n \t\t}\n \t\tpriv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job;\n@@ -2678,6 +2810,9 @@ flow_hw_actions_validate(struct rte_eth_dev *dev,\n \t\t\tif (ret < 0)\n \t\t\t\treturn ret;\n \t\t\tbreak;\n+\t\tcase RTE_FLOW_ACTION_TYPE_COUNT:\n+\t\t\t/* TODO: Validation logic */\n+\t\t\tbreak;\n \t\tcase RTE_FLOW_ACTION_TYPE_END:\n \t\t\tactions_end = true;\n \t\t\tbreak;\n@@ -4355,6 +4490,12 @@ flow_hw_configure(struct rte_eth_dev *dev,\n \t}\n \tif (_queue_attr)\n \t\tmlx5_free(_queue_attr);\n+\tif (port_attr->nb_counters) {\n+\t\tpriv->hws_cpool = mlx5_hws_cnt_pool_create(dev, port_attr,\n+\t\t\t\tnb_queue);\n+\t\tif (priv->hws_cpool == NULL)\n+\t\t\tgoto err;\n+\t}\n \treturn 0;\n err:\n \tflow_hw_free_vport_actions(priv);\n@@ -4424,6 +4565,8 @@ flow_hw_resource_release(struct rte_eth_dev *dev)\n \t\tmlx5_ipool_destroy(priv->acts_ipool);\n \t\tpriv->acts_ipool = NULL;\n \t}\n+\tif (priv->hws_cpool)\n+\t\tmlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);\n \tmlx5_free(priv->hw_q);\n \tpriv->hw_q = NULL;\n \tclaim_zero(mlx5dr_context_close(priv->dr_ctx));\n@@ -4565,10 +4708,28 @@ flow_hw_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,\n \t\t\t     void *user_data,\n \t\t\t     struct rte_flow_error *error)\n {\n+\tstruct rte_flow_action_handle *handle = NULL;\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tcnt_id_t cnt_id;\n+\n \tRTE_SET_USED(queue);\n \tRTE_SET_USED(attr);\n \tRTE_SET_USED(user_data);\n-\treturn flow_dv_action_create(dev, conf, action, error);\n+\tswitch (action->type) {\n+\tcase RTE_FLOW_ACTION_TYPE_COUNT:\n+\t\tif (mlx5_hws_cnt_shared_get(priv->hws_cpool, &cnt_id))\n+\t\t\trte_flow_error_set(error, ENODEV,\n+\t\t\t\t\tRTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t\tNULL,\n+\t\t\t\t\t\"counter are not configured!\");\n+\t\telse\n+\t\t\thandle = (struct rte_flow_action_handle *)\n+\t\t\t\t (uintptr_t)cnt_id;\n+\t\tbreak;\n+\tdefault:\n+\t\thandle = flow_dv_action_create(dev, conf, action, error);\n+\t}\n+\treturn handle;\n }\n \n /**\n@@ -4632,10 +4793,172 @@ flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,\n \t\t\t      void *user_data,\n \t\t\t      struct rte_flow_error *error)\n {\n+\tuint32_t act_idx = (uint32_t)(uintptr_t)handle;\n+\tuint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\n \tRTE_SET_USED(queue);\n \tRTE_SET_USED(attr);\n \tRTE_SET_USED(user_data);\n-\treturn flow_dv_action_destroy(dev, handle, error);\n+\tswitch (type) {\n+\tcase MLX5_INDIRECT_ACTION_TYPE_COUNT:\n+\t\treturn mlx5_hws_cnt_shared_put(priv->hws_cpool, &act_idx);\n+\tdefault:\n+\t\treturn flow_dv_action_destroy(dev, handle, error);\n+\t}\n+}\n+\n+static int\n+flow_hw_query_counter(const struct rte_eth_dev *dev, uint32_t counter,\n+\t\t      void *data, struct rte_flow_error *error)\n+{\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_hws_cnt *cnt;\n+\tstruct rte_flow_query_count *qc = data;\n+\tuint32_t iidx = mlx5_hws_cnt_iidx(priv->hws_cpool, counter);\n+\tuint64_t pkts, bytes;\n+\n+\tif (!mlx5_hws_cnt_id_valid(counter))\n+\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\t\tRTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,\n+\t\t\t\t\"counter are not available\");\n+\tcnt = &priv->hws_cpool->pool[iidx];\n+\t__hws_cnt_query_raw(priv->hws_cpool, counter, &pkts, &bytes);\n+\tqc->hits_set = 1;\n+\tqc->bytes_set = 1;\n+\tqc->hits = pkts - cnt->reset.hits;\n+\tqc->bytes = bytes - cnt->reset.bytes;\n+\tif (qc->reset) {\n+\t\tcnt->reset.bytes = bytes;\n+\t\tcnt->reset.hits = pkts;\n+\t}\n+\treturn 0;\n+}\n+\n+static int\n+flow_hw_query(struct rte_eth_dev *dev,\n+\t      struct rte_flow *flow __rte_unused,\n+\t      const struct rte_flow_action *actions __rte_unused,\n+\t      void *data __rte_unused,\n+\t      struct rte_flow_error *error __rte_unused)\n+{\n+\tint ret = -EINVAL;\n+\tstruct rte_flow_hw *hw_flow = (struct rte_flow_hw *)flow;\n+\n+\tfor (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {\n+\t\tswitch (actions->type) {\n+\t\tcase RTE_FLOW_ACTION_TYPE_VOID:\n+\t\t\tbreak;\n+\t\tcase RTE_FLOW_ACTION_TYPE_COUNT:\n+\t\t\tret = flow_hw_query_counter(dev, hw_flow->cnt_id, data,\n+\t\t\t\t\t\t  error);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\treturn rte_flow_error_set(error, ENOTSUP,\n+\t\t\t\t\t\t  RTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t\t\t  actions,\n+\t\t\t\t\t\t  \"action not supported\");\n+\t\t}\n+\t}\n+\treturn ret;\n+}\n+\n+/**\n+ * Create indirect action.\n+ *\n+ * @param[in] dev\n+ *   Pointer to the Ethernet device structure.\n+ * @param[in] conf\n+ *   Shared action configuration.\n+ * @param[in] action\n+ *   Action specification used to create indirect action.\n+ * @param[out] error\n+ *   Perform verbose error reporting if not NULL. Initialized in case of\n+ *   error only.\n+ *\n+ * @return\n+ *   A valid shared action handle in case of success, NULL otherwise and\n+ *   rte_errno is set.\n+ */\n+static struct rte_flow_action_handle *\n+flow_hw_action_create(struct rte_eth_dev *dev,\n+\t\t       const struct rte_flow_indir_action_conf *conf,\n+\t\t       const struct rte_flow_action *action,\n+\t\t       struct rte_flow_error *err)\n+{\n+\treturn flow_hw_action_handle_create(dev, UINT32_MAX, NULL, conf, action,\n+\t\t\t\t\t    NULL, err);\n+}\n+\n+/**\n+ * Destroy the indirect action.\n+ * Release action related resources on the NIC and the memory.\n+ * Lock free, (mutex should be acquired by caller).\n+ * Dispatcher for action type specific call.\n+ *\n+ * @param[in] dev\n+ *   Pointer to the Ethernet device structure.\n+ * @param[in] handle\n+ *   The indirect action object handle to be removed.\n+ * @param[out] error\n+ *   Perform verbose error reporting if not NULL. Initialized in case of\n+ *   error only.\n+ *\n+ * @return\n+ *   0 on success, otherwise negative errno value.\n+ */\n+static int\n+flow_hw_action_destroy(struct rte_eth_dev *dev,\n+\t\t       struct rte_flow_action_handle *handle,\n+\t\t       struct rte_flow_error *error)\n+{\n+\treturn flow_hw_action_handle_destroy(dev, UINT32_MAX, NULL, handle,\n+\t\t\tNULL, error);\n+}\n+\n+/**\n+ * Updates in place shared action configuration.\n+ *\n+ * @param[in] dev\n+ *   Pointer to the Ethernet device structure.\n+ * @param[in] handle\n+ *   The indirect action object handle to be updated.\n+ * @param[in] update\n+ *   Action specification used to modify the action pointed by *handle*.\n+ *   *update* could be of same type with the action pointed by the *handle*\n+ *   handle argument, or some other structures like a wrapper, depending on\n+ *   the indirect action type.\n+ * @param[out] error\n+ *   Perform verbose error reporting if not NULL. Initialized in case of\n+ *   error only.\n+ *\n+ * @return\n+ *   0 on success, otherwise negative errno value.\n+ */\n+static int\n+flow_hw_action_update(struct rte_eth_dev *dev,\n+\t\t      struct rte_flow_action_handle *handle,\n+\t\t      const void *update,\n+\t\t      struct rte_flow_error *err)\n+{\n+\treturn flow_hw_action_handle_update(dev, UINT32_MAX, NULL, handle,\n+\t\t\tupdate, NULL, err);\n+}\n+\n+static int\n+flow_hw_action_query(struct rte_eth_dev *dev,\n+\t\t     const struct rte_flow_action_handle *handle, void *data,\n+\t\t     struct rte_flow_error *error)\n+{\n+\tuint32_t act_idx = (uint32_t)(uintptr_t)handle;\n+\tuint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;\n+\n+\tswitch (type) {\n+\tcase MLX5_INDIRECT_ACTION_TYPE_COUNT:\n+\t\treturn flow_hw_query_counter(dev, act_idx, data, error);\n+\tdefault:\n+\t\treturn flow_dv_action_query(dev, handle, data, error);\n+\t}\n }\n \n const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {\n@@ -4657,10 +4980,11 @@ const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {\n \t.async_action_destroy = flow_hw_action_handle_destroy,\n \t.async_action_update = flow_hw_action_handle_update,\n \t.action_validate = flow_dv_action_validate,\n-\t.action_create = flow_dv_action_create,\n-\t.action_destroy = flow_dv_action_destroy,\n-\t.action_update = flow_dv_action_update,\n-\t.action_query = flow_dv_action_query,\n+\t.action_create = flow_hw_action_create,\n+\t.action_destroy = flow_hw_action_destroy,\n+\t.action_update = flow_hw_action_update,\n+\t.action_query = flow_hw_action_query,\n+\t.query = flow_hw_query,\n };\n \n /**\ndiff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c\nnew file mode 100644\nindex 0000000000..e2408ef36d\n--- /dev/null\n+++ b/drivers/net/mlx5/mlx5_hws_cnt.c\n@@ -0,0 +1,528 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2020 Mellanox Technologies, Ltd\n+ */\n+\n+#include <stdint.h>\n+#include <rte_malloc.h>\n+#include <mlx5_malloc.h>\n+#include <rte_ring.h>\n+#include <mlx5_devx_cmds.h>\n+#include <rte_cycles.h>\n+\n+#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)\n+\n+#include \"mlx5_utils.h\"\n+#include \"mlx5_hws_cnt.h\"\n+\n+#define HWS_CNT_CACHE_SZ_DEFAULT 511\n+#define HWS_CNT_CACHE_PRELOAD_DEFAULT 254\n+#define HWS_CNT_CACHE_FETCH_DEFAULT 254\n+#define HWS_CNT_CACHE_THRESHOLD_DEFAULT 254\n+#define HWS_CNT_ALLOC_FACTOR_DEFAULT 20\n+\n+static void\n+__hws_cnt_id_load(struct mlx5_hws_cnt_pool *cpool)\n+{\n+\tuint32_t preload;\n+\tuint32_t q_num = cpool->cache->q_num;\n+\tuint32_t cnt_num = mlx5_hws_cnt_pool_get_size(cpool);\n+\tcnt_id_t cnt_id, iidx = 0;\n+\tuint32_t qidx;\n+\tstruct rte_ring *qcache = NULL;\n+\n+\t/*\n+\t * Counter ID order is important for tracking the max number of in used\n+\t * counter for querying, which means counter internal index order must\n+\t * be from zero to the number user configured, i.e: 0 - 8000000.\n+\t * Need to load counter ID in this order into the cache firstly,\n+\t * and then the global free list.\n+\t * In the end, user fetch the the counter from minimal to the maximum.\n+\t */\n+\tpreload = RTE_MIN(cpool->cache->preload_sz, cnt_num / q_num);\n+\tfor (qidx = 0; qidx < q_num; qidx++) {\n+\t\tfor (; iidx < preload * (qidx + 1); iidx++) {\n+\t\t\tcnt_id = mlx5_hws_cnt_id_gen(cpool, iidx);\n+\t\t\tqcache = cpool->cache->qcache[qidx];\n+\t\t\tif (qcache)\n+\t\t\t\trte_ring_enqueue_elem(qcache, &cnt_id,\n+\t\t\t\t\t\tsizeof(cnt_id));\n+\t\t}\n+\t}\n+\tfor (; iidx < cnt_num; iidx++) {\n+\t\tcnt_id = mlx5_hws_cnt_id_gen(cpool, iidx);\n+\t\trte_ring_enqueue_elem(cpool->free_list, &cnt_id,\n+\t\t\t\tsizeof(cnt_id));\n+\t}\n+}\n+\n+static void\n+__mlx5_hws_cnt_svc(struct mlx5_dev_ctx_shared *sh,\n+\t\tstruct mlx5_hws_cnt_pool *cpool)\n+{\n+\tstruct rte_ring *reset_list = cpool->wait_reset_list;\n+\tstruct rte_ring *reuse_list = cpool->reuse_list;\n+\tuint32_t reset_cnt_num;\n+\tstruct rte_ring_zc_data zcdr = {0};\n+\tstruct rte_ring_zc_data zcdu = {0};\n+\n+\treset_cnt_num = rte_ring_count(reset_list);\n+\tdo {\n+\t\tcpool->query_gen++;\n+\t\tmlx5_aso_cnt_query(sh, cpool);\n+\t\tzcdr.n1 = 0;\n+\t\tzcdu.n1 = 0;\n+\t\trte_ring_enqueue_zc_burst_elem_start(reuse_list,\n+\t\t\t\tsizeof(cnt_id_t), reset_cnt_num, &zcdu,\n+\t\t\t\tNULL);\n+\t\trte_ring_dequeue_zc_burst_elem_start(reset_list,\n+\t\t\t\tsizeof(cnt_id_t), reset_cnt_num, &zcdr,\n+\t\t\t\tNULL);\n+\t\t__hws_cnt_r2rcpy(&zcdu, &zcdr, reset_cnt_num);\n+\t\trte_ring_dequeue_zc_elem_finish(reset_list,\n+\t\t\t\treset_cnt_num);\n+\t\trte_ring_enqueue_zc_elem_finish(reuse_list,\n+\t\t\t\treset_cnt_num);\n+\t\treset_cnt_num = rte_ring_count(reset_list);\n+\t} while (reset_cnt_num > 0);\n+}\n+\n+static void\n+mlx5_hws_cnt_raw_data_free(struct mlx5_dev_ctx_shared *sh,\n+\t\t\t   struct mlx5_hws_cnt_raw_data_mng *mng)\n+{\n+\tif (mng == NULL)\n+\t\treturn;\n+\tsh->cdev->mr_scache.dereg_mr_cb(&mng->mr);\n+\tmlx5_free(mng->raw);\n+\tmlx5_free(mng);\n+}\n+\n+__rte_unused\n+static struct mlx5_hws_cnt_raw_data_mng *\n+mlx5_hws_cnt_raw_data_alloc(struct mlx5_dev_ctx_shared *sh, uint32_t n)\n+{\n+\tstruct mlx5_hws_cnt_raw_data_mng *mng = NULL;\n+\tint ret;\n+\tsize_t sz = n * sizeof(struct flow_counter_stats);\n+\n+\tmng = mlx5_malloc(MLX5_MEM_ANY | MLX5_MEM_ZERO, sizeof(*mng), 0,\n+\t\t\tSOCKET_ID_ANY);\n+\tif (mng == NULL)\n+\t\tgoto error;\n+\tmng->raw = mlx5_malloc(MLX5_MEM_ANY | MLX5_MEM_ZERO, sz, 0,\n+\t\t\tSOCKET_ID_ANY);\n+\tif (mng->raw == NULL)\n+\t\tgoto error;\n+\tret = sh->cdev->mr_scache.reg_mr_cb(sh->cdev->pd, mng->raw, sz,\n+\t\t\t\t\t    &mng->mr);\n+\tif (ret) {\n+\t\trte_errno = errno;\n+\t\tgoto error;\n+\t}\n+\treturn mng;\n+error:\n+\tmlx5_hws_cnt_raw_data_free(sh, mng);\n+\treturn NULL;\n+}\n+\n+static void *\n+mlx5_hws_cnt_svc(void *opaque)\n+{\n+\tstruct mlx5_dev_ctx_shared *sh =\n+\t\t(struct mlx5_dev_ctx_shared *)opaque;\n+\tuint64_t interval =\n+\t\t(uint64_t)sh->cnt_svc->query_interval * (US_PER_S / MS_PER_S);\n+\tuint16_t port_id;\n+\tuint64_t start_cycle, query_cycle = 0;\n+\tuint64_t query_us;\n+\tuint64_t sleep_us;\n+\n+\twhile (sh->cnt_svc->svc_running != 0) {\n+\t\tstart_cycle = rte_rdtsc();\n+\t\tMLX5_ETH_FOREACH_DEV(port_id, sh->cdev->dev) {\n+\t\t\tstruct mlx5_priv *opriv =\n+\t\t\t\trte_eth_devices[port_id].data->dev_private;\n+\t\t\tif (opriv != NULL &&\n+\t\t\t    opriv->sh == sh &&\n+\t\t\t    opriv->hws_cpool != NULL) {\n+\t\t\t\t__mlx5_hws_cnt_svc(sh, opriv->hws_cpool);\n+\t\t\t}\n+\t\t}\n+\t\tquery_cycle = rte_rdtsc() - start_cycle;\n+\t\tquery_us = query_cycle / (rte_get_timer_hz() / US_PER_S);\n+\t\tsleep_us = interval - query_us;\n+\t\tif (interval > query_us)\n+\t\t\trte_delay_us_sleep(sleep_us);\n+\t}\n+\treturn NULL;\n+}\n+\n+struct mlx5_hws_cnt_pool *\n+mlx5_hws_cnt_pool_init(const struct mlx5_hws_cnt_pool_cfg *pcfg,\n+\t\tconst struct mlx5_hws_cache_param *ccfg)\n+{\n+\tchar mz_name[RTE_MEMZONE_NAMESIZE];\n+\tstruct mlx5_hws_cnt_pool *cntp;\n+\tuint64_t cnt_num = 0;\n+\tuint32_t qidx;\n+\n+\tMLX5_ASSERT(pcfg);\n+\tMLX5_ASSERT(ccfg);\n+\tcntp = mlx5_malloc(MLX5_MEM_ANY | MLX5_MEM_ZERO, sizeof(*cntp), 0,\n+\t\t\t   SOCKET_ID_ANY);\n+\tif (cntp == NULL)\n+\t\treturn NULL;\n+\n+\tcntp->cfg = *pcfg;\n+\tcntp->cache = mlx5_malloc(MLX5_MEM_ANY | MLX5_MEM_ZERO,\n+\t\t\tsizeof(*cntp->cache) +\n+\t\t\tsizeof(((struct mlx5_hws_cnt_pool_caches *)0)->qcache[0])\n+\t\t\t\t* ccfg->q_num, 0, SOCKET_ID_ANY);\n+\tif (cntp->cache == NULL)\n+\t\tgoto error;\n+\t /* store the necessary cache parameters. */\n+\tcntp->cache->fetch_sz = ccfg->fetch_sz;\n+\tcntp->cache->preload_sz = ccfg->preload_sz;\n+\tcntp->cache->threshold = ccfg->threshold;\n+\tcntp->cache->q_num = ccfg->q_num;\n+\tcnt_num = pcfg->request_num * (100 + pcfg->alloc_factor) / 100;\n+\tif (cnt_num > UINT32_MAX) {\n+\t\tDRV_LOG(ERR, \"counter number %\"PRIu64\" is out of 32bit range\",\n+\t\t\tcnt_num);\n+\t\tgoto error;\n+\t}\n+\tcntp->pool = mlx5_malloc(MLX5_MEM_ANY | MLX5_MEM_ZERO,\n+\t\t\tsizeof(struct mlx5_hws_cnt) *\n+\t\t\tpcfg->request_num * (100 + pcfg->alloc_factor) / 100,\n+\t\t\t0, SOCKET_ID_ANY);\n+\tif (cntp->pool == NULL)\n+\t\tgoto error;\n+\tsnprintf(mz_name, sizeof(mz_name), \"%s_F_RING\", pcfg->name);\n+\tcntp->free_list = rte_ring_create_elem(mz_name, sizeof(cnt_id_t),\n+\t\t\t(uint32_t)cnt_num, SOCKET_ID_ANY,\n+\t\t\tRING_F_SP_ENQ | RING_F_MC_HTS_DEQ | RING_F_EXACT_SZ);\n+\tif (cntp->free_list == NULL) {\n+\t\tDRV_LOG(ERR, \"failed to create free list ring\");\n+\t\tgoto error;\n+\t}\n+\tsnprintf(mz_name, sizeof(mz_name), \"%s_R_RING\", pcfg->name);\n+\tcntp->wait_reset_list = rte_ring_create_elem(mz_name, sizeof(cnt_id_t),\n+\t\t\t(uint32_t)cnt_num, SOCKET_ID_ANY,\n+\t\t\tRING_F_MP_HTS_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);\n+\tif (cntp->wait_reset_list == NULL) {\n+\t\tDRV_LOG(ERR, \"failed to create free list ring\");\n+\t\tgoto error;\n+\t}\n+\tsnprintf(mz_name, sizeof(mz_name), \"%s_U_RING\", pcfg->name);\n+\tcntp->reuse_list = rte_ring_create_elem(mz_name, sizeof(cnt_id_t),\n+\t\t\t(uint32_t)cnt_num, SOCKET_ID_ANY,\n+\t\t\tRING_F_SP_ENQ | RING_F_MC_HTS_DEQ | RING_F_EXACT_SZ);\n+\tif (cntp->reuse_list == NULL) {\n+\t\tDRV_LOG(ERR, \"failed to create reuse list ring\");\n+\t\tgoto error;\n+\t}\n+\tfor (qidx = 0; qidx < ccfg->q_num; qidx++) {\n+\t\tsnprintf(mz_name, sizeof(mz_name), \"%s_cache/%u\", pcfg->name,\n+\t\t\t\tqidx);\n+\t\tcntp->cache->qcache[qidx] = rte_ring_create(mz_name, ccfg->size,\n+\t\t\t\tSOCKET_ID_ANY,\n+\t\t\t\tRING_F_SP_ENQ | RING_F_SC_DEQ |\n+\t\t\t\tRING_F_EXACT_SZ);\n+\t\tif (cntp->cache->qcache[qidx] == NULL)\n+\t\t\tgoto error;\n+\t}\n+\treturn cntp;\n+error:\n+\tmlx5_hws_cnt_pool_deinit(cntp);\n+\treturn NULL;\n+}\n+\n+void\n+mlx5_hws_cnt_pool_deinit(struct mlx5_hws_cnt_pool * const cntp)\n+{\n+\tuint32_t qidx = 0;\n+\tif (cntp == NULL)\n+\t\treturn;\n+\trte_ring_free(cntp->free_list);\n+\trte_ring_free(cntp->wait_reset_list);\n+\trte_ring_free(cntp->reuse_list);\n+\tif (cntp->cache) {\n+\t\tfor (qidx = 0; qidx < cntp->cache->q_num; qidx++)\n+\t\t\trte_ring_free(cntp->cache->qcache[qidx]);\n+\t}\n+\tmlx5_free(cntp->cache);\n+\tmlx5_free(cntp->raw_mng);\n+\tmlx5_free(cntp->pool);\n+\tmlx5_free(cntp);\n+}\n+\n+int\n+mlx5_hws_cnt_service_thread_create(struct mlx5_dev_ctx_shared *sh)\n+{\n+#define CNT_THREAD_NAME_MAX 256\n+\tchar name[CNT_THREAD_NAME_MAX];\n+\trte_cpuset_t cpuset;\n+\tint ret;\n+\tuint32_t service_core = sh->cnt_svc->service_core;\n+\n+\tCPU_ZERO(&cpuset);\n+\tsh->cnt_svc->svc_running = 1;\n+\tret = pthread_create(&sh->cnt_svc->service_thread, NULL,\n+\t\t\tmlx5_hws_cnt_svc, sh);\n+\tif (ret != 0) {\n+\t\tDRV_LOG(ERR, \"Failed to create HW steering's counter service thread.\");\n+\t\treturn -ENOSYS;\n+\t}\n+\tsnprintf(name, CNT_THREAD_NAME_MAX - 1, \"%s/svc@%d\",\n+\t\t sh->ibdev_name, service_core);\n+\trte_thread_setname(sh->cnt_svc->service_thread, name);\n+\tCPU_SET(service_core, &cpuset);\n+\tpthread_setaffinity_np(sh->cnt_svc->service_thread, sizeof(cpuset),\n+\t\t\t\t&cpuset);\n+\treturn 0;\n+}\n+\n+void\n+mlx5_hws_cnt_service_thread_destroy(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tif (sh->cnt_svc->service_thread == 0)\n+\t\treturn;\n+\tsh->cnt_svc->svc_running = 0;\n+\tpthread_join(sh->cnt_svc->service_thread, NULL);\n+\tsh->cnt_svc->service_thread = 0;\n+}\n+\n+int\n+mlx5_hws_cnt_pool_dcs_alloc(struct mlx5_dev_ctx_shared *sh,\n+\t\t\t    struct mlx5_hws_cnt_pool *cpool)\n+{\n+\tstruct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr;\n+\tuint32_t max_log_bulk_sz = 0;\n+\tuint32_t log_bulk_sz;\n+\tuint32_t idx, alloced = 0;\n+\tunsigned int cnt_num = mlx5_hws_cnt_pool_get_size(cpool);\n+\tstruct mlx5_devx_counter_attr attr = {0};\n+\tstruct mlx5_devx_obj *dcs;\n+\n+\tif (hca_attr->flow_counter_bulk_log_max_alloc == 0) {\n+\t\tDRV_LOG(ERR,\n+\t\t\t\"Fw doesn't support bulk log max alloc\");\n+\t\treturn -1;\n+\t}\n+\tmax_log_bulk_sz = 23; /* hard code to 8M (1 << 23). */\n+\tcnt_num = RTE_ALIGN_CEIL(cnt_num, 4); /* minimal 4 counter in bulk. */\n+\tlog_bulk_sz = RTE_MIN(max_log_bulk_sz, rte_log2_u32(cnt_num));\n+\tattr.pd = sh->cdev->pdn;\n+\tattr.pd_valid = 1;\n+\tattr.bulk_log_max_alloc = 1;\n+\tattr.flow_counter_bulk_log_size = log_bulk_sz;\n+\tidx = 0;\n+\tdcs = mlx5_devx_cmd_flow_counter_alloc_general(sh->cdev->ctx, &attr);\n+\tif (dcs == NULL)\n+\t\tgoto error;\n+\tcpool->dcs_mng.dcs[idx].obj = dcs;\n+\tcpool->dcs_mng.dcs[idx].batch_sz = (1 << log_bulk_sz);\n+\tcpool->dcs_mng.batch_total++;\n+\tidx++;\n+\tcpool->dcs_mng.dcs[0].iidx = 0;\n+\talloced = cpool->dcs_mng.dcs[0].batch_sz;\n+\tif (cnt_num > cpool->dcs_mng.dcs[0].batch_sz) {\n+\t\tfor (; idx < MLX5_HWS_CNT_DCS_NUM; idx++) {\n+\t\t\tattr.flow_counter_bulk_log_size = --max_log_bulk_sz;\n+\t\t\tdcs = mlx5_devx_cmd_flow_counter_alloc_general\n+\t\t\t\t(sh->cdev->ctx, &attr);\n+\t\t\tif (dcs == NULL)\n+\t\t\t\tgoto error;\n+\t\t\tcpool->dcs_mng.dcs[idx].obj = dcs;\n+\t\t\tcpool->dcs_mng.dcs[idx].batch_sz =\n+\t\t\t\t(1 << max_log_bulk_sz);\n+\t\t\tcpool->dcs_mng.dcs[idx].iidx = alloced;\n+\t\t\talloced += cpool->dcs_mng.dcs[idx].batch_sz;\n+\t\t\tcpool->dcs_mng.batch_total++;\n+\t\t}\n+\t}\n+\treturn 0;\n+error:\n+\tDRV_LOG(DEBUG,\n+\t\t\"Cannot alloc device counter, allocated[%\" PRIu32 \"] request[%\" PRIu32 \"]\",\n+\t\talloced, cnt_num);\n+\tfor (idx = 0; idx < cpool->dcs_mng.batch_total; idx++) {\n+\t\tmlx5_devx_cmd_destroy(cpool->dcs_mng.dcs[idx].obj);\n+\t\tcpool->dcs_mng.dcs[idx].obj = NULL;\n+\t\tcpool->dcs_mng.dcs[idx].batch_sz = 0;\n+\t\tcpool->dcs_mng.dcs[idx].iidx = 0;\n+\t}\n+\tcpool->dcs_mng.batch_total = 0;\n+\treturn -1;\n+}\n+\n+void\n+mlx5_hws_cnt_pool_dcs_free(struct mlx5_dev_ctx_shared *sh,\n+\t\t\t   struct mlx5_hws_cnt_pool *cpool)\n+{\n+\tuint32_t idx;\n+\n+\tif (cpool == NULL)\n+\t\treturn;\n+\tfor (idx = 0; idx < MLX5_HWS_CNT_DCS_NUM; idx++)\n+\t\tmlx5_devx_cmd_destroy(cpool->dcs_mng.dcs[idx].obj);\n+\tif (cpool->raw_mng) {\n+\t\tmlx5_hws_cnt_raw_data_free(sh, cpool->raw_mng);\n+\t\tcpool->raw_mng = NULL;\n+\t}\n+}\n+\n+int\n+mlx5_hws_cnt_pool_action_create(struct mlx5_priv *priv,\n+\t\tstruct mlx5_hws_cnt_pool *cpool)\n+{\n+\tuint32_t idx;\n+\tint ret = 0;\n+\tstruct mlx5_hws_cnt_dcs *dcs;\n+\tuint32_t flags;\n+\n+\tflags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX;\n+\tif (priv->sh->config.dv_esw_en && priv->master)\n+\t\tflags |= MLX5DR_ACTION_FLAG_HWS_FDB;\n+\tfor (idx = 0; idx < cpool->dcs_mng.batch_total; idx++) {\n+\t\tdcs = &cpool->dcs_mng.dcs[idx];\n+\t\tdcs->dr_action = mlx5dr_action_create_counter(priv->dr_ctx,\n+\t\t\t\t\t(struct mlx5dr_devx_obj *)dcs->obj,\n+\t\t\t\t\tflags);\n+\t\tif (dcs->dr_action == NULL) {\n+\t\t\tmlx5_hws_cnt_pool_action_destroy(cpool);\n+\t\t\tret = -ENOSYS;\n+\t\t\tbreak;\n+\t\t}\n+\t}\n+\treturn ret;\n+}\n+\n+void\n+mlx5_hws_cnt_pool_action_destroy(struct mlx5_hws_cnt_pool *cpool)\n+{\n+\tuint32_t idx;\n+\tstruct mlx5_hws_cnt_dcs *dcs;\n+\n+\tfor (idx = 0; idx < cpool->dcs_mng.batch_total; idx++) {\n+\t\tdcs = &cpool->dcs_mng.dcs[idx];\n+\t\tif (dcs->dr_action != NULL) {\n+\t\t\tmlx5dr_action_destroy(dcs->dr_action);\n+\t\t\tdcs->dr_action = NULL;\n+\t\t}\n+\t}\n+}\n+\n+struct mlx5_hws_cnt_pool *\n+mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev,\n+\t\tconst struct rte_flow_port_attr *pattr, uint16_t nb_queue)\n+{\n+\tstruct mlx5_hws_cnt_pool *cpool = NULL;\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_hws_cache_param cparam = {0};\n+\tstruct mlx5_hws_cnt_pool_cfg pcfg = {0};\n+\tchar *mp_name;\n+\tint ret = 0;\n+\tsize_t sz;\n+\n+\t/* init cnt service if not. */\n+\tif (priv->sh->cnt_svc == NULL) {\n+\t\tret = mlx5_hws_cnt_svc_init(priv->sh);\n+\t\tif (ret != 0)\n+\t\t\treturn NULL;\n+\t}\n+\tcparam.fetch_sz = HWS_CNT_CACHE_FETCH_DEFAULT;\n+\tcparam.preload_sz = HWS_CNT_CACHE_PRELOAD_DEFAULT;\n+\tcparam.q_num = nb_queue;\n+\tcparam.threshold = HWS_CNT_CACHE_THRESHOLD_DEFAULT;\n+\tcparam.size = HWS_CNT_CACHE_SZ_DEFAULT;\n+\tpcfg.alloc_factor = HWS_CNT_ALLOC_FACTOR_DEFAULT;\n+\tmp_name = mlx5_malloc(MLX5_MEM_ZERO, RTE_MEMZONE_NAMESIZE, 0,\n+\t\t\tSOCKET_ID_ANY);\n+\tif (mp_name == NULL)\n+\t\tgoto error;\n+\tsnprintf(mp_name, RTE_MEMZONE_NAMESIZE, \"MLX5_HWS_CNT_POOL_%u\",\n+\t\t\tdev->data->port_id);\n+\tpcfg.name = mp_name;\n+\tpcfg.request_num = pattr->nb_counters;\n+\tcpool = mlx5_hws_cnt_pool_init(&pcfg, &cparam);\n+\tif (cpool == NULL)\n+\t\tgoto error;\n+\tret = mlx5_hws_cnt_pool_dcs_alloc(priv->sh, cpool);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\tsz = RTE_ALIGN_CEIL(mlx5_hws_cnt_pool_get_size(cpool), 4);\n+\tcpool->raw_mng = mlx5_hws_cnt_raw_data_alloc(priv->sh, sz);\n+\tif (cpool->raw_mng == NULL)\n+\t\tgoto error;\n+\t__hws_cnt_id_load(cpool);\n+\t/*\n+\t * Bump query gen right after pool create so the\n+\t * pre-loaded counters can be used directly\n+\t * because they already have init value no need\n+\t * to wait for query.\n+\t */\n+\tcpool->query_gen = 1;\n+\tret = mlx5_hws_cnt_pool_action_create(priv, cpool);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\tpriv->sh->cnt_svc->refcnt++;\n+\treturn cpool;\n+error:\n+\tmlx5_hws_cnt_pool_destroy(priv->sh, cpool);\n+\treturn NULL;\n+}\n+\n+void\n+mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,\n+\t\tstruct mlx5_hws_cnt_pool *cpool)\n+{\n+\tif (cpool == NULL)\n+\t\treturn;\n+\tif (--sh->cnt_svc->refcnt == 0)\n+\t\tmlx5_hws_cnt_svc_deinit(sh);\n+\tmlx5_hws_cnt_pool_action_destroy(cpool);\n+\tmlx5_hws_cnt_pool_dcs_free(sh, cpool);\n+\tmlx5_hws_cnt_raw_data_free(sh, cpool->raw_mng);\n+\tmlx5_free((void *)cpool->cfg.name);\n+\tmlx5_hws_cnt_pool_deinit(cpool);\n+}\n+\n+int\n+mlx5_hws_cnt_svc_init(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tint ret;\n+\n+\tsh->cnt_svc = mlx5_malloc(MLX5_MEM_ANY | MLX5_MEM_ZERO,\n+\t\t\tsizeof(*sh->cnt_svc), 0, SOCKET_ID_ANY);\n+\tif (sh->cnt_svc == NULL)\n+\t\treturn -1;\n+\tsh->cnt_svc->query_interval = sh->config.cnt_svc.cycle_time;\n+\tsh->cnt_svc->service_core = sh->config.cnt_svc.service_core;\n+\tret = mlx5_aso_cnt_queue_init(sh);\n+\tif (ret != 0) {\n+\t\tmlx5_free(sh->cnt_svc);\n+\t\tsh->cnt_svc = NULL;\n+\t\treturn -1;\n+\t}\n+\tret = mlx5_hws_cnt_service_thread_create(sh);\n+\tif (ret != 0) {\n+\t\tmlx5_aso_cnt_queue_uninit(sh);\n+\t\tmlx5_free(sh->cnt_svc);\n+\t\tsh->cnt_svc = NULL;\n+\t}\n+\treturn 0;\n+}\n+\n+void\n+mlx5_hws_cnt_svc_deinit(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tif (sh->cnt_svc == NULL)\n+\t\treturn;\n+\tmlx5_hws_cnt_service_thread_destroy(sh);\n+\tmlx5_aso_cnt_queue_uninit(sh);\n+\tmlx5_free(sh->cnt_svc);\n+\tsh->cnt_svc = NULL;\n+}\n+\n+#endif\ndiff --git a/drivers/net/mlx5/mlx5_hws_cnt.h b/drivers/net/mlx5/mlx5_hws_cnt.h\nnew file mode 100644\nindex 0000000000..5fab4ba597\n--- /dev/null\n+++ b/drivers/net/mlx5/mlx5_hws_cnt.h\n@@ -0,0 +1,558 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2022 Mellanox Technologies, Ltd\n+ */\n+\n+#ifndef _MLX5_HWS_CNT_H_\n+#define _MLX5_HWS_CNT_H_\n+\n+#include <rte_ring.h>\n+#include \"mlx5_utils.h\"\n+#include \"mlx5_flow.h\"\n+\n+/*\n+ * COUNTER ID's layout\n+ *       3                   2                   1                   0\n+ *     1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0\n+ *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n+ *    | T |       | D |                                               |\n+ *    ~ Y |       | C |                    IDX                        ~\n+ *    | P |       | S |                                               |\n+ *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n+ *\n+ *    Bit 31:30 = TYPE = MLX5_INDIRECT_ACTION_TYPE_COUNT = b'10\n+ *    Bit 25:24 = DCS index\n+ *    Bit 23:00 = IDX in this counter belonged DCS bulk.\n+ */\n+typedef uint32_t cnt_id_t;\n+\n+#define MLX5_HWS_CNT_DCS_NUM 4\n+#define MLX5_HWS_CNT_DCS_IDX_OFFSET 24\n+#define MLX5_HWS_CNT_DCS_IDX_MASK 0x3\n+#define MLX5_HWS_CNT_IDX_MASK ((1UL << MLX5_HWS_CNT_DCS_IDX_OFFSET) - 1)\n+\n+struct mlx5_hws_cnt_dcs {\n+\tvoid *dr_action;\n+\tuint32_t batch_sz;\n+\tuint32_t iidx; /* internal index of first counter in this bulk. */\n+\tstruct mlx5_devx_obj *obj;\n+};\n+\n+struct mlx5_hws_cnt_dcs_mng {\n+\tuint32_t batch_total;\n+\tstruct mlx5_hws_cnt_dcs dcs[MLX5_HWS_CNT_DCS_NUM];\n+};\n+\n+struct mlx5_hws_cnt {\n+\tstruct flow_counter_stats reset;\n+\tunion {\n+\t\tuint32_t share: 1;\n+\t\t/*\n+\t\t * share will be set to 1 when this counter is used as indirect\n+\t\t * action. Only meaningful when user own this counter.\n+\t\t */\n+\t\tuint32_t query_gen_when_free;\n+\t\t/*\n+\t\t * When PMD own this counter (user put back counter to PMD\n+\t\t * counter pool, i.e), this field recorded value of counter\n+\t\t * pools query generation at time user release the counter.\n+\t\t */\n+\t};\n+};\n+\n+struct mlx5_hws_cnt_raw_data_mng {\n+\tstruct flow_counter_stats *raw;\n+\tstruct mlx5_pmd_mr mr;\n+};\n+\n+struct mlx5_hws_cache_param {\n+\tuint32_t size;\n+\tuint32_t q_num;\n+\tuint32_t fetch_sz;\n+\tuint32_t threshold;\n+\tuint32_t preload_sz;\n+};\n+\n+struct mlx5_hws_cnt_pool_cfg {\n+\tchar *name;\n+\tuint32_t request_num;\n+\tuint32_t alloc_factor;\n+};\n+\n+struct mlx5_hws_cnt_pool_caches {\n+\tuint32_t fetch_sz;\n+\tuint32_t threshold;\n+\tuint32_t preload_sz;\n+\tuint32_t q_num;\n+\tstruct rte_ring *qcache[];\n+};\n+\n+struct mlx5_hws_cnt_pool {\n+\tstruct mlx5_hws_cnt_pool_cfg cfg __rte_cache_aligned;\n+\tstruct mlx5_hws_cnt_dcs_mng dcs_mng __rte_cache_aligned;\n+\tuint32_t query_gen __rte_cache_aligned;\n+\tstruct mlx5_hws_cnt *pool;\n+\tstruct mlx5_hws_cnt_raw_data_mng *raw_mng;\n+\tstruct rte_ring *reuse_list;\n+\tstruct rte_ring *free_list;\n+\tstruct rte_ring *wait_reset_list;\n+\tstruct mlx5_hws_cnt_pool_caches *cache;\n+} __rte_cache_aligned;\n+\n+/**\n+ * Translate counter id into internal index (start from 0), which can be used\n+ * as index of raw/cnt pool.\n+ *\n+ * @param cnt_id\n+ *   The external counter id\n+ * @return\n+ *   Internal index\n+ */\n+static __rte_always_inline cnt_id_t\n+mlx5_hws_cnt_iidx(struct mlx5_hws_cnt_pool *cpool, cnt_id_t cnt_id)\n+{\n+\tuint8_t dcs_idx = cnt_id >> MLX5_HWS_CNT_DCS_IDX_OFFSET;\n+\tuint32_t offset = cnt_id & MLX5_HWS_CNT_IDX_MASK;\n+\n+\tdcs_idx &= MLX5_HWS_CNT_DCS_IDX_MASK;\n+\treturn (cpool->dcs_mng.dcs[dcs_idx].iidx + offset);\n+}\n+\n+/**\n+ * Check if it's valid counter id.\n+ */\n+static __rte_always_inline bool\n+mlx5_hws_cnt_id_valid(cnt_id_t cnt_id)\n+{\n+\treturn (cnt_id >> MLX5_INDIRECT_ACTION_TYPE_OFFSET) ==\n+\t\tMLX5_INDIRECT_ACTION_TYPE_COUNT ? true : false;\n+}\n+\n+/**\n+ * Generate Counter id from internal index.\n+ *\n+ * @param cpool\n+ *   The pointer to counter pool\n+ * @param index\n+ *   The internal counter index.\n+ *\n+ * @return\n+ *   Counter id\n+ */\n+static __rte_always_inline cnt_id_t\n+mlx5_hws_cnt_id_gen(struct mlx5_hws_cnt_pool *cpool, cnt_id_t iidx)\n+{\n+\tstruct mlx5_hws_cnt_dcs_mng *dcs_mng = &cpool->dcs_mng;\n+\tuint32_t idx;\n+\tuint32_t offset;\n+\tcnt_id_t cnt_id;\n+\n+\tfor (idx = 0, offset = iidx; idx < dcs_mng->batch_total; idx++) {\n+\t\tif (dcs_mng->dcs[idx].batch_sz <= offset)\n+\t\t\toffset -= dcs_mng->dcs[idx].batch_sz;\n+\t\telse\n+\t\t\tbreak;\n+\t}\n+\tcnt_id = offset;\n+\tcnt_id |= (idx << MLX5_HWS_CNT_DCS_IDX_OFFSET);\n+\treturn (MLX5_INDIRECT_ACTION_TYPE_COUNT <<\n+\t\t\tMLX5_INDIRECT_ACTION_TYPE_OFFSET) | cnt_id;\n+}\n+\n+static __rte_always_inline void\n+__hws_cnt_query_raw(struct mlx5_hws_cnt_pool *cpool, cnt_id_t cnt_id,\n+\t\tuint64_t *raw_pkts, uint64_t *raw_bytes)\n+{\n+\tstruct mlx5_hws_cnt_raw_data_mng *raw_mng = cpool->raw_mng;\n+\tstruct flow_counter_stats s[2];\n+\tuint8_t i = 0x1;\n+\tsize_t stat_sz = sizeof(s[0]);\n+\tuint32_t iidx = mlx5_hws_cnt_iidx(cpool, cnt_id);\n+\n+\tmemcpy(&s[0], &raw_mng->raw[iidx], stat_sz);\n+\tdo {\n+\t\tmemcpy(&s[i & 1], &raw_mng->raw[iidx], stat_sz);\n+\t\tif (memcmp(&s[0], &s[1], stat_sz) == 0) {\n+\t\t\t*raw_pkts = rte_be_to_cpu_64(s[0].hits);\n+\t\t\t*raw_bytes = rte_be_to_cpu_64(s[0].bytes);\n+\t\t\tbreak;\n+\t\t}\n+\t\ti = ~i;\n+\t} while (1);\n+}\n+\n+/**\n+ * Copy elems from one zero-copy ring to zero-copy ring in place.\n+ *\n+ * The input is a rte ring zero-copy data struct, which has two pointer.\n+ * in case of the wrapper happened, the ptr2 will be meaningful.\n+ *\n+ * So this rountin needs to consider the situation that the address given by\n+ * source and destination could be both wrapped.\n+ * First, calculate the first number of element needs to be copied until wrapped\n+ * address, which could be in source or destination.\n+ * Second, copy left number of element until second wrapped address. If in first\n+ * step the wrapped address is source, then this time it must be in destination.\n+ * and vice-vers.\n+ * Third, copy all left numbe of element.\n+ *\n+ * In worst case, we need copy three pieces of continuous memory.\n+ *\n+ * @param zcdd\n+ *   A pointer to zero-copy data of dest ring.\n+ * @param zcds\n+ *   A pointer to zero-copy data of source ring.\n+ * @param n\n+ *   Number of elems to copy.\n+ */\n+static __rte_always_inline void\n+__hws_cnt_r2rcpy(struct rte_ring_zc_data *zcdd, struct rte_ring_zc_data *zcds,\n+\t\tunsigned int n)\n+{\n+\tunsigned int n1, n2, n3;\n+\tvoid *s1, *s2, *s3;\n+\tvoid *d1, *d2, *d3;\n+\n+\ts1 = zcds->ptr1;\n+\td1 = zcdd->ptr1;\n+\tn1 = RTE_MIN(zcdd->n1, zcds->n1);\n+\tif (zcds->n1 > n1) {\n+\t\tn2 = zcds->n1 - n1;\n+\t\ts2 = RTE_PTR_ADD(zcds->ptr1, sizeof(cnt_id_t) * n1);\n+\t\td2 = zcdd->ptr2;\n+\t\tn3 = n - n1 - n2;\n+\t\ts3 = zcds->ptr2;\n+\t\td3 = RTE_PTR_ADD(zcdd->ptr2, sizeof(cnt_id_t) * n2);\n+\t} else {\n+\t\tn2 = zcdd->n1 - n1;\n+\t\ts2 = zcds->ptr2;\n+\t\td2 = RTE_PTR_ADD(zcdd->ptr1, sizeof(cnt_id_t) * n1);\n+\t\tn3 = n - n1 - n2;\n+\t\ts3 = RTE_PTR_ADD(zcds->ptr2, sizeof(cnt_id_t) * n2);\n+\t\td3 = zcdd->ptr2;\n+\t}\n+\tmemcpy(d1, s1, n1 * sizeof(cnt_id_t));\n+\tif (n2 != 0) {\n+\t\tmemcpy(d2, s2, n2 * sizeof(cnt_id_t));\n+\t\tif (n3 != 0)\n+\t\t\tmemcpy(d3, s3, n3 * sizeof(cnt_id_t));\n+\t}\n+}\n+\n+static __rte_always_inline int\n+mlx5_hws_cnt_pool_cache_flush(struct mlx5_hws_cnt_pool *cpool,\n+\t\t\t      uint32_t queue_id)\n+{\n+\tunsigned int ret;\n+\tstruct rte_ring_zc_data zcdr = {0};\n+\tstruct rte_ring_zc_data zcdc = {0};\n+\tstruct rte_ring *reset_list = NULL;\n+\tstruct rte_ring *qcache = cpool->cache->qcache[queue_id];\n+\n+\tret = rte_ring_dequeue_zc_burst_elem_start(qcache,\n+\t\t\tsizeof(cnt_id_t), rte_ring_count(qcache), &zcdc,\n+\t\t\tNULL);\n+\tMLX5_ASSERT(ret);\n+\treset_list = cpool->wait_reset_list;\n+\trte_ring_enqueue_zc_burst_elem_start(reset_list,\n+\t\t\tsizeof(cnt_id_t), ret, &zcdr, NULL);\n+\t__hws_cnt_r2rcpy(&zcdr, &zcdc, ret);\n+\trte_ring_enqueue_zc_elem_finish(reset_list, ret);\n+\trte_ring_dequeue_zc_elem_finish(qcache, ret);\n+\treturn 0;\n+}\n+\n+static __rte_always_inline int\n+mlx5_hws_cnt_pool_cache_fetch(struct mlx5_hws_cnt_pool *cpool,\n+\t\t\t      uint32_t queue_id)\n+{\n+\tstruct rte_ring *qcache = cpool->cache->qcache[queue_id];\n+\tstruct rte_ring *free_list = NULL;\n+\tstruct rte_ring *reuse_list = NULL;\n+\tstruct rte_ring *list = NULL;\n+\tstruct rte_ring_zc_data zcdf = {0};\n+\tstruct rte_ring_zc_data zcdc = {0};\n+\tstruct rte_ring_zc_data zcdu = {0};\n+\tstruct rte_ring_zc_data zcds = {0};\n+\tstruct mlx5_hws_cnt_pool_caches *cache = cpool->cache;\n+\tunsigned int ret;\n+\n+\treuse_list = cpool->reuse_list;\n+\tret = rte_ring_dequeue_zc_burst_elem_start(reuse_list,\n+\t\t\tsizeof(cnt_id_t), cache->fetch_sz, &zcdu, NULL);\n+\tzcds = zcdu;\n+\tlist = reuse_list;\n+\tif (unlikely(ret == 0)) { /* no reuse counter. */\n+\t\trte_ring_dequeue_zc_elem_finish(reuse_list, 0);\n+\t\tfree_list = cpool->free_list;\n+\t\tret = rte_ring_dequeue_zc_burst_elem_start(free_list,\n+\t\t\t\tsizeof(cnt_id_t), cache->fetch_sz, &zcdf, NULL);\n+\t\tzcds = zcdf;\n+\t\tlist = free_list;\n+\t\tif (unlikely(ret == 0)) { /* no free counter. */\n+\t\t\trte_ring_dequeue_zc_elem_finish(free_list, 0);\n+\t\t\tif (rte_ring_count(cpool->wait_reset_list))\n+\t\t\t\treturn -EAGAIN;\n+\t\t\treturn -ENOENT;\n+\t\t}\n+\t}\n+\trte_ring_enqueue_zc_burst_elem_start(qcache, sizeof(cnt_id_t),\n+\t\t\tret, &zcdc, NULL);\n+\t__hws_cnt_r2rcpy(&zcdc, &zcds, ret);\n+\trte_ring_dequeue_zc_elem_finish(list, ret);\n+\trte_ring_enqueue_zc_elem_finish(qcache, ret);\n+\treturn 0;\n+}\n+\n+static __rte_always_inline int\n+__mlx5_hws_cnt_pool_enqueue_revert(struct rte_ring *r, unsigned int n,\n+\t\tstruct rte_ring_zc_data *zcd)\n+{\n+\tuint32_t current_head = 0;\n+\tuint32_t revert2head = 0;\n+\n+\tMLX5_ASSERT(r->prod.sync_type == RTE_RING_SYNC_ST);\n+\tMLX5_ASSERT(r->cons.sync_type == RTE_RING_SYNC_ST);\n+\tcurrent_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);\n+\tMLX5_ASSERT(n <= r->capacity);\n+\tMLX5_ASSERT(n <= rte_ring_count(r));\n+\trevert2head = current_head - n;\n+\tr->prod.head = revert2head; /* This ring should be SP. */\n+\t__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,\n+\t\t\t&zcd->ptr1, &zcd->n1, &zcd->ptr2);\n+\t/* Update tail */\n+\t__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);\n+\treturn n;\n+}\n+\n+/**\n+ * Put one counter back in the mempool.\n+ *\n+ * @param cpool\n+ *   A pointer to the counter pool structure.\n+ * @param cnt_id\n+ *   A counter id to be added.\n+ * @return\n+ *   - 0: Success; object taken\n+ *   - -ENOENT: not enough entry in pool\n+ */\n+static __rte_always_inline int\n+mlx5_hws_cnt_pool_put(struct mlx5_hws_cnt_pool *cpool,\n+\t\tuint32_t *queue, cnt_id_t *cnt_id)\n+{\n+\tunsigned int ret = 0;\n+\tstruct rte_ring_zc_data zcdc = {0};\n+\tstruct rte_ring_zc_data zcdr = {0};\n+\tstruct rte_ring *qcache = NULL;\n+\tunsigned int wb_num = 0; /* cache write-back number. */\n+\tcnt_id_t iidx;\n+\n+\tiidx = mlx5_hws_cnt_iidx(cpool, *cnt_id);\n+\tcpool->pool[iidx].query_gen_when_free =\n+\t\t__atomic_load_n(&cpool->query_gen, __ATOMIC_RELAXED);\n+\tif (likely(queue != NULL))\n+\t\tqcache = cpool->cache->qcache[*queue];\n+\tif (unlikely(qcache == NULL)) {\n+\t\tret = rte_ring_enqueue_elem(cpool->wait_reset_list, cnt_id,\n+\t\t\t\tsizeof(cnt_id_t));\n+\t\tMLX5_ASSERT(ret == 0);\n+\t\treturn ret;\n+\t}\n+\tret = rte_ring_enqueue_burst_elem(qcache, cnt_id, sizeof(cnt_id_t), 1,\n+\t\t\t\t\t  NULL);\n+\tif (unlikely(ret == 0)) { /* cache is full. */\n+\t\twb_num = rte_ring_count(qcache) - cpool->cache->threshold;\n+\t\tMLX5_ASSERT(wb_num < rte_ring_count(qcache));\n+\t\t__mlx5_hws_cnt_pool_enqueue_revert(qcache, wb_num, &zcdc);\n+\t\trte_ring_enqueue_zc_burst_elem_start(cpool->wait_reset_list,\n+\t\t\t\tsizeof(cnt_id_t), wb_num, &zcdr, NULL);\n+\t\t__hws_cnt_r2rcpy(&zcdr, &zcdc, wb_num);\n+\t\trte_ring_enqueue_zc_elem_finish(cpool->wait_reset_list, wb_num);\n+\t\t/* write-back THIS counter too */\n+\t\tret = rte_ring_enqueue_burst_elem(cpool->wait_reset_list,\n+\t\t\t\tcnt_id, sizeof(cnt_id_t), 1, NULL);\n+\t}\n+\treturn ret == 1 ? 0 : -ENOENT;\n+}\n+\n+/**\n+ * Get one counter from the pool.\n+ *\n+ * If @param queue is not null, objects will be retrieved first from queue's\n+ * cache, subsequently from the common pool. Note that it can return -ENOENT\n+ * when the local cache and common pool are empty, even if cache from other\n+ * queue are full.\n+ *\n+ * @param cntp\n+ *   A pointer to the counter pool structure.\n+ * @param queue\n+ *   A pointer to HWS queue. If null, it means fetch from common pool.\n+ * @param cnt_id\n+ *   A pointer to a cnt_id_t * pointer (counter id) that will be filled.\n+ * @return\n+ *   - 0: Success; objects taken.\n+ *   - -ENOENT: Not enough entries in the mempool; no object is retrieved.\n+ *   - -EAGAIN: counter is not ready; try again.\n+ */\n+static __rte_always_inline int\n+mlx5_hws_cnt_pool_get(struct mlx5_hws_cnt_pool *cpool,\n+\t\tuint32_t *queue, cnt_id_t *cnt_id)\n+{\n+\tunsigned int ret;\n+\tstruct rte_ring_zc_data zcdc = {0};\n+\tstruct rte_ring *qcache = NULL;\n+\tuint32_t query_gen = 0;\n+\tcnt_id_t iidx, tmp_cid = 0;\n+\n+\tif (likely(queue != NULL))\n+\t\tqcache = cpool->cache->qcache[*queue];\n+\tif (unlikely(qcache == NULL)) {\n+\t\tret = rte_ring_dequeue_elem(cpool->reuse_list, &tmp_cid,\n+\t\t\t\tsizeof(cnt_id_t));\n+\t\tif (unlikely(ret != 0)) {\n+\t\t\tret = rte_ring_dequeue_elem(cpool->free_list, &tmp_cid,\n+\t\t\t\t\tsizeof(cnt_id_t));\n+\t\t\tif (unlikely(ret != 0)) {\n+\t\t\t\tif (rte_ring_count(cpool->wait_reset_list))\n+\t\t\t\t\treturn -EAGAIN;\n+\t\t\t\treturn -ENOENT;\n+\t\t\t}\n+\t\t}\n+\t\t*cnt_id = tmp_cid;\n+\t\tiidx = mlx5_hws_cnt_iidx(cpool, *cnt_id);\n+\t\t__hws_cnt_query_raw(cpool, *cnt_id,\n+\t\t\t\t    &cpool->pool[iidx].reset.hits,\n+\t\t\t\t    &cpool->pool[iidx].reset.bytes);\n+\t\treturn 0;\n+\t}\n+\tret = rte_ring_dequeue_zc_burst_elem_start(qcache, sizeof(cnt_id_t), 1,\n+\t\t\t&zcdc, NULL);\n+\tif (unlikely(ret == 0)) { /* local cache is empty. */\n+\t\trte_ring_dequeue_zc_elem_finish(qcache, 0);\n+\t\t/* let's fetch from global free list. */\n+\t\tret = mlx5_hws_cnt_pool_cache_fetch(cpool, *queue);\n+\t\tif (unlikely(ret != 0))\n+\t\t\treturn ret;\n+\t\trte_ring_dequeue_zc_burst_elem_start(qcache, sizeof(cnt_id_t),\n+\t\t\t\t1, &zcdc, NULL);\n+\t}\n+\t/* get one from local cache. */\n+\t*cnt_id = (*(cnt_id_t *)zcdc.ptr1);\n+\tiidx = mlx5_hws_cnt_iidx(cpool, *cnt_id);\n+\tquery_gen = cpool->pool[iidx].query_gen_when_free;\n+\tif (cpool->query_gen == query_gen) { /* counter is waiting to reset. */\n+\t\trte_ring_dequeue_zc_elem_finish(qcache, 0);\n+\t\t/* write-back counter to reset list. */\n+\t\tmlx5_hws_cnt_pool_cache_flush(cpool, *queue);\n+\t\t/* let's fetch from global free list. */\n+\t\tret = mlx5_hws_cnt_pool_cache_fetch(cpool, *queue);\n+\t\tif (unlikely(ret != 0))\n+\t\t\treturn ret;\n+\t\trte_ring_dequeue_zc_burst_elem_start(qcache, sizeof(cnt_id_t),\n+\t\t\t\t1, &zcdc, NULL);\n+\t\t*cnt_id = *(cnt_id_t *)zcdc.ptr1;\n+\t}\n+\t__hws_cnt_query_raw(cpool, *cnt_id, &cpool->pool[iidx].reset.hits,\n+\t\t\t    &cpool->pool[iidx].reset.bytes);\n+\trte_ring_dequeue_zc_elem_finish(qcache, 1);\n+\tcpool->pool[iidx].share = 0;\n+\treturn 0;\n+}\n+\n+static __rte_always_inline unsigned int\n+mlx5_hws_cnt_pool_get_size(struct mlx5_hws_cnt_pool *cpool)\n+{\n+\treturn rte_ring_get_capacity(cpool->free_list);\n+}\n+\n+static __rte_always_inline int\n+mlx5_hws_cnt_pool_get_action_offset(struct mlx5_hws_cnt_pool *cpool,\n+\t\tcnt_id_t cnt_id, struct mlx5dr_action **action,\n+\t\tuint32_t *offset)\n+{\n+\tuint8_t idx = cnt_id >> MLX5_HWS_CNT_DCS_IDX_OFFSET;\n+\n+\tidx &= MLX5_HWS_CNT_DCS_IDX_MASK;\n+\t*action = cpool->dcs_mng.dcs[idx].dr_action;\n+\t*offset = cnt_id & MLX5_HWS_CNT_IDX_MASK;\n+\treturn 0;\n+}\n+\n+static __rte_always_inline int\n+mlx5_hws_cnt_shared_get(struct mlx5_hws_cnt_pool *cpool, cnt_id_t *cnt_id)\n+{\n+\tint ret;\n+\tuint32_t iidx;\n+\n+\tret = mlx5_hws_cnt_pool_get(cpool, NULL, cnt_id);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\tiidx = mlx5_hws_cnt_iidx(cpool, *cnt_id);\n+\tMLX5_ASSERT(cpool->pool[iidx].share == 0);\n+\tcpool->pool[iidx].share = 1;\n+\treturn 0;\n+}\n+\n+static __rte_always_inline int\n+mlx5_hws_cnt_shared_put(struct mlx5_hws_cnt_pool *cpool, cnt_id_t *cnt_id)\n+{\n+\tint ret;\n+\tuint32_t iidx = mlx5_hws_cnt_iidx(cpool, *cnt_id);\n+\n+\tcpool->pool[iidx].share = 0;\n+\tret = mlx5_hws_cnt_pool_put(cpool, NULL, cnt_id);\n+\tif (unlikely(ret != 0))\n+\t\tcpool->pool[iidx].share = 1; /* fail to release, restore. */\n+\treturn ret;\n+}\n+\n+static __rte_always_inline bool\n+mlx5_hws_cnt_is_shared(struct mlx5_hws_cnt_pool *cpool, cnt_id_t cnt_id)\n+{\n+\tuint32_t iidx = mlx5_hws_cnt_iidx(cpool, cnt_id);\n+\n+\treturn cpool->pool[iidx].share ? true : false;\n+}\n+\n+/* init HWS counter pool. */\n+struct mlx5_hws_cnt_pool *\n+mlx5_hws_cnt_pool_init(const struct mlx5_hws_cnt_pool_cfg *pcfg,\n+\t\tconst struct mlx5_hws_cache_param *ccfg);\n+\n+void\n+mlx5_hws_cnt_pool_deinit(struct mlx5_hws_cnt_pool *cntp);\n+\n+int\n+mlx5_hws_cnt_service_thread_create(struct mlx5_dev_ctx_shared *sh);\n+\n+void\n+mlx5_hws_cnt_service_thread_destroy(struct mlx5_dev_ctx_shared *sh);\n+\n+int\n+mlx5_hws_cnt_pool_dcs_alloc(struct mlx5_dev_ctx_shared *sh,\n+\t\tstruct mlx5_hws_cnt_pool *cpool);\n+void\n+mlx5_hws_cnt_pool_dcs_free(struct mlx5_dev_ctx_shared *sh,\n+\t\tstruct mlx5_hws_cnt_pool *cpool);\n+\n+int\n+mlx5_hws_cnt_pool_action_create(struct mlx5_priv *priv,\n+\t\tstruct mlx5_hws_cnt_pool *cpool);\n+\n+void\n+mlx5_hws_cnt_pool_action_destroy(struct mlx5_hws_cnt_pool *cpool);\n+\n+struct mlx5_hws_cnt_pool *\n+mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev,\n+\t\tconst struct rte_flow_port_attr *pattr, uint16_t nb_queue);\n+\n+void\n+mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,\n+\t\tstruct mlx5_hws_cnt_pool *cpool);\n+\n+int\n+mlx5_hws_cnt_svc_init(struct mlx5_dev_ctx_shared *sh);\n+\n+void\n+mlx5_hws_cnt_svc_deinit(struct mlx5_dev_ctx_shared *sh);\n+\n+#endif /* _MLX5_HWS_CNT_H_ */\n",
    "prefixes": [
        "v3",
        "08/17"
    ]
}