Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/76389/?format=api
{ "id": 76389, "url": "http://patches.dpdk.org/api/patches/76389/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1599128029-2092-6-git-send-email-michaelba@nvidia.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1599128029-2092-6-git-send-email-michaelba@nvidia.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1599128029-2092-6-git-send-email-michaelba@nvidia.com", "date": "2020-09-03T10:13:36", "name": "[v1,05/18] net/mlx5: separate Rx queue object creations", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "f8cf0d593ee2485d0dd1f82850555a95f51ae8e2", "submitter": { "id": 1949, "url": "http://patches.dpdk.org/api/people/1949/?format=api", "name": "Michael Baum", "email": "michaelba@nvidia.com" }, "delegate": { "id": 3268, "url": "http://patches.dpdk.org/api/users/3268/?format=api", "username": "rasland", "first_name": "Raslan", "last_name": "Darawsheh", "email": "rasland@nvidia.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1599128029-2092-6-git-send-email-michaelba@nvidia.com/mbox/", "series": [ { "id": 11924, "url": "http://patches.dpdk.org/api/series/11924/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=11924", "date": "2020-09-03T10:13:31", "name": "mlx5 Rx DevX/Verbs separation", "version": 1, "mbox": "http://patches.dpdk.org/series/11924/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/76389/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/76389/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 63AF1A04DB;\n\tThu, 3 Sep 2020 12:15:33 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id DE3911C11B;\n\tThu, 3 Sep 2020 12:14:48 +0200 (CEST)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n by dpdk.org (Postfix) with ESMTP id E37921C0D1\n for <dev@dpdk.org>; Thu, 3 Sep 2020 12:14:45 +0200 (CEST)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n michaelba@nvidia.com) with SMTP; 3 Sep 2020 13:14:42 +0300", "from nvidia.com (pegasus07.mtr.labs.mlnx [10.210.16.112])\n by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 083AEP93031645;\n Thu, 3 Sep 2020 13:14:41 +0300" ], "From": "Michael Baum <michaelba@nvidia.com>", "To": "dev@dpdk.org", "Cc": "Matan Azrad <matan@nvidia.com>, Raslan Darawsheh <rasland@nvidia.com>,\n Viacheslav Ovsiienko <viacheslavo@nvidia.com>", "Date": "Thu, 3 Sep 2020 10:13:36 +0000", "Message-Id": "<1599128029-2092-6-git-send-email-michaelba@nvidia.com>", "X-Mailer": "git-send-email 1.8.3.1", "In-Reply-To": "<1599128029-2092-1-git-send-email-michaelba@nvidia.com>", "References": "<1599128029-2092-1-git-send-email-michaelba@nvidia.com>", "Subject": "[dpdk-dev] [PATCH v1 05/18] net/mlx5: separate Rx queue object\n\tcreations", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "As an arrangement to Windows OS support, the Verbs operations should be\nseparated to another file.\nBy this way, the build can easily cut the unsupported Verbs APIs from\nthe compilation process.\n\nDefine operation structure and DevX module in addition to the existing\nlinux Verbs module.\nSeparate Rx object creation into the Verbs/DevX modules and update the\noperation structure according to the OS support and the user\nconfiguration.\n\nSigned-off-by: Michael Baum <michaelba@nvidia.com>\nAcked-by: Matan Azrad <matan@nvidia.com>\n---\n drivers/net/mlx5/Makefile | 1 +\n drivers/net/mlx5/linux/mlx5_os.c | 5 +\n drivers/net/mlx5/linux/mlx5_verbs.c | 344 ++++++++++++++\n drivers/net/mlx5/linux/mlx5_verbs.h | 4 +\n drivers/net/mlx5/meson.build | 1 +\n drivers/net/mlx5/mlx5.h | 31 ++\n drivers/net/mlx5/mlx5_devx.c | 562 +++++++++++++++++++++++\n drivers/net/mlx5/mlx5_rxq.c | 861 +-----------------------------------\n drivers/net/mlx5/mlx5_rxtx.h | 31 +-\n drivers/net/mlx5/mlx5_trigger.c | 45 +-\n 10 files changed, 963 insertions(+), 922 deletions(-)", "diff": "diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile\nindex 6097688..50ad3bc 100644\n--- a/drivers/net/mlx5/Makefile\n+++ b/drivers/net/mlx5/Makefile\n@@ -31,6 +31,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_meter.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_dv.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_verbs.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_utils.c\n+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_devx.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += linux/mlx5_socket.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += linux/mlx5_os.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += linux/mlx5_ethdev_os.c\ndiff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c\nindex bf1f82b..694fbd3 100644\n--- a/drivers/net/mlx5/linux/mlx5_os.c\n+++ b/drivers/net/mlx5/linux/mlx5_os.c\n@@ -46,6 +46,7 @@\n #include \"rte_pmd_mlx5.h\"\n #include \"mlx5_verbs.h\"\n #include \"mlx5_nl.h\"\n+#include \"mlx5_devx.h\"\n \n #define MLX5_TAGS_HLIST_ARRAY_SIZE 8192\n \n@@ -1322,6 +1323,10 @@\n \t\t\tgoto error;\n \t\t}\n \t}\n+\tif (config->devx && config->dv_flow_en)\n+\t\tpriv->obj_ops = &devx_obj_ops;\n+\telse\n+\t\tpriv->obj_ops = &ibv_obj_ops;\n \treturn eth_dev;\n error:\n \tif (priv) {\ndiff --git a/drivers/net/mlx5/linux/mlx5_verbs.c b/drivers/net/mlx5/linux/mlx5_verbs.c\nindex 6271f0f..ff71513 100644\n--- a/drivers/net/mlx5/linux/mlx5_verbs.c\n+++ b/drivers/net/mlx5/linux/mlx5_verbs.c\n@@ -8,6 +8,7 @@\n #include <stdint.h>\n #include <unistd.h>\n #include <inttypes.h>\n+#include <sys/queue.h>\n \n #include \"mlx5_autoconf.h\"\n \n@@ -21,6 +22,10 @@\n #include <mlx5_common_mr.h>\n #include <mlx5_rxtx.h>\n #include <mlx5_verbs.h>\n+#include <mlx5_rxtx.h>\n+#include <mlx5_utils.h>\n+#include <mlx5_malloc.h>\n+\n /**\n * Register mr. Given protection domain pointer, pointer to addr and length\n * register the memory region.\n@@ -86,6 +91,345 @@\n \treturn mlx5_glue->modify_wq(rxq_obj->wq, &mod);\n }\n \n+/**\n+ * Create a CQ Verbs object.\n+ *\n+ * @param dev\n+ * Pointer to Ethernet device.\n+ * @param priv\n+ * Pointer to device private data.\n+ * @param rxq_data\n+ * Pointer to Rx queue data.\n+ * @param cqe_n\n+ * Number of CQEs in CQ.\n+ * @param rxq_obj\n+ * Pointer to Rx queue object data.\n+ *\n+ * @return\n+ * The Verbs object initialized, NULL otherwise and rte_errno is set.\n+ */\n+static struct ibv_cq *\n+mlx5_ibv_cq_new(struct rte_eth_dev *dev, struct mlx5_priv *priv,\n+\t\tstruct mlx5_rxq_data *rxq_data,\n+\t\tunsigned int cqe_n, struct mlx5_rxq_obj *rxq_obj)\n+{\n+\tstruct {\n+\t\tstruct ibv_cq_init_attr_ex ibv;\n+\t\tstruct mlx5dv_cq_init_attr mlx5;\n+\t} cq_attr;\n+\n+\tcq_attr.ibv = (struct ibv_cq_init_attr_ex){\n+\t\t.cqe = cqe_n,\n+\t\t.channel = rxq_obj->ibv_channel,\n+\t\t.comp_mask = 0,\n+\t};\n+\tcq_attr.mlx5 = (struct mlx5dv_cq_init_attr){\n+\t\t.comp_mask = 0,\n+\t};\n+\tif (priv->config.cqe_comp && !rxq_data->hw_timestamp) {\n+\t\tcq_attr.mlx5.comp_mask |=\n+\t\t\t\tMLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;\n+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT\n+\t\tcq_attr.mlx5.cqe_comp_res_format =\n+\t\t\t\tmlx5_rxq_mprq_enabled(rxq_data) ?\n+\t\t\t\tMLX5DV_CQE_RES_FORMAT_CSUM_STRIDX :\n+\t\t\t\tMLX5DV_CQE_RES_FORMAT_HASH;\n+#else\n+\t\tcq_attr.mlx5.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;\n+#endif\n+\t\t/*\n+\t\t * For vectorized Rx, it must not be doubled in order to\n+\t\t * make cq_ci and rq_ci aligned.\n+\t\t */\n+\t\tif (mlx5_rxq_check_vec_support(rxq_data) < 0)\n+\t\t\tcq_attr.ibv.cqe *= 2;\n+\t} else if (priv->config.cqe_comp && rxq_data->hw_timestamp) {\n+\t\tDRV_LOG(DEBUG,\n+\t\t\t\"Port %u Rx CQE compression is disabled for HW\"\n+\t\t\t\" timestamp.\",\n+\t\t\tdev->data->port_id);\n+\t}\n+#ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD\n+\tif (priv->config.cqe_pad) {\n+\t\tcq_attr.mlx5.comp_mask |= MLX5DV_CQ_INIT_ATTR_MASK_FLAGS;\n+\t\tcq_attr.mlx5.flags |= MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD;\n+\t}\n+#endif\n+\treturn mlx5_glue->cq_ex_to_cq(mlx5_glue->dv_create_cq(priv->sh->ctx,\n+\t\t\t\t\t\t\t &cq_attr.ibv,\n+\t\t\t\t\t\t\t &cq_attr.mlx5));\n+}\n+\n+/**\n+ * Create a WQ Verbs object.\n+ *\n+ * @param dev\n+ * Pointer to Ethernet device.\n+ * @param priv\n+ * Pointer to device private data.\n+ * @param rxq_data\n+ * Pointer to Rx queue data.\n+ * @param idx\n+ * Queue index in DPDK Rx queue array.\n+ * @param wqe_n\n+ * Number of WQEs in WQ.\n+ * @param rxq_obj\n+ * Pointer to Rx queue object data.\n+ *\n+ * @return\n+ * The Verbs object initialized, NULL otherwise and rte_errno is set.\n+ */\n+static struct ibv_wq *\n+mlx5_ibv_wq_new(struct rte_eth_dev *dev, struct mlx5_priv *priv,\n+\t\tstruct mlx5_rxq_data *rxq_data, uint16_t idx,\n+\t\tunsigned int wqe_n, struct mlx5_rxq_obj *rxq_obj)\n+{\n+\tstruct {\n+\t\tstruct ibv_wq_init_attr ibv;\n+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT\n+\t\tstruct mlx5dv_wq_init_attr mlx5;\n+#endif\n+\t} wq_attr;\n+\n+\twq_attr.ibv = (struct ibv_wq_init_attr){\n+\t\t.wq_context = NULL, /* Could be useful in the future. */\n+\t\t.wq_type = IBV_WQT_RQ,\n+\t\t/* Max number of outstanding WRs. */\n+\t\t.max_wr = wqe_n >> rxq_data->sges_n,\n+\t\t/* Max number of scatter/gather elements in a WR. */\n+\t\t.max_sge = 1 << rxq_data->sges_n,\n+\t\t.pd = priv->sh->pd,\n+\t\t.cq = rxq_obj->ibv_cq,\n+\t\t.comp_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING | 0,\n+\t\t.create_flags = (rxq_data->vlan_strip ?\n+\t\t\t\t IBV_WQ_FLAGS_CVLAN_STRIPPING : 0),\n+\t};\n+\t/* By default, FCS (CRC) is stripped by hardware. */\n+\tif (rxq_data->crc_present) {\n+\t\twq_attr.ibv.create_flags |= IBV_WQ_FLAGS_SCATTER_FCS;\n+\t\twq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;\n+\t}\n+\tif (priv->config.hw_padding) {\n+#if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)\n+\t\twq_attr.ibv.create_flags |= IBV_WQ_FLAG_RX_END_PADDING;\n+\t\twq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;\n+#elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)\n+\t\twq_attr.ibv.create_flags |= IBV_WQ_FLAGS_PCI_WRITE_END_PADDING;\n+\t\twq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;\n+#endif\n+\t}\n+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT\n+\twq_attr.mlx5 = (struct mlx5dv_wq_init_attr){\n+\t\t.comp_mask = 0,\n+\t};\n+\tif (mlx5_rxq_mprq_enabled(rxq_data)) {\n+\t\tstruct mlx5dv_striding_rq_init_attr *mprq_attr =\n+\t\t\t\t\t\t&wq_attr.mlx5.striding_rq_attrs;\n+\n+\t\twq_attr.mlx5.comp_mask |= MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ;\n+\t\t*mprq_attr = (struct mlx5dv_striding_rq_init_attr){\n+\t\t\t.single_stride_log_num_of_bytes = rxq_data->strd_sz_n,\n+\t\t\t.single_wqe_log_num_of_strides = rxq_data->strd_num_n,\n+\t\t\t.two_byte_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT,\n+\t\t};\n+\t}\n+\trxq_obj->wq = mlx5_glue->dv_create_wq(priv->sh->ctx, &wq_attr.ibv,\n+\t\t\t\t\t &wq_attr.mlx5);\n+#else\n+\trxq_obj->wq = mlx5_glue->create_wq(priv->sh->ctx, &wq_attr.ibv);\n+#endif\n+\tif (rxq_obj->wq) {\n+\t\t/*\n+\t\t * Make sure number of WRs*SGEs match expectations since a queue\n+\t\t * cannot allocate more than \"desc\" buffers.\n+\t\t */\n+\t\tif (wq_attr.ibv.max_wr != (wqe_n >> rxq_data->sges_n) ||\n+\t\t wq_attr.ibv.max_sge != (1u << rxq_data->sges_n)) {\n+\t\t\tDRV_LOG(ERR,\n+\t\t\t\t\"Port %u Rx queue %u requested %u*%u but got\"\n+\t\t\t\t\" %u*%u WRs*SGEs.\",\n+\t\t\t\tdev->data->port_id, idx,\n+\t\t\t\twqe_n >> rxq_data->sges_n,\n+\t\t\t\t(1 << rxq_data->sges_n),\n+\t\t\t\twq_attr.ibv.max_wr, wq_attr.ibv.max_sge);\n+\t\t\tclaim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));\n+\t\t\trxq_obj->wq = NULL;\n+\t\t\trte_errno = EINVAL;\n+\t\t}\n+\t}\n+\treturn rxq_obj->wq;\n+}\n+\n+/**\n+ * Create the Rx queue Verbs object.\n+ *\n+ * @param dev\n+ * Pointer to Ethernet device.\n+ * @param idx\n+ * Queue index in DPDK Rx queue array.\n+ *\n+ * @return\n+ * The Verbs object initialized, NULL otherwise and rte_errno is set.\n+ */\n+static struct mlx5_rxq_obj *\n+mlx5_rxq_ibv_obj_new(struct rte_eth_dev *dev, uint16_t idx)\n+{\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n+\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n+\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n+\tstruct ibv_wq_attr mod;\n+\tunsigned int cqe_n;\n+\tunsigned int wqe_n = 1 << rxq_data->elts_n;\n+\tstruct mlx5_rxq_obj *tmpl = NULL;\n+\tstruct mlx5dv_cq cq_info;\n+\tstruct mlx5dv_rwq rwq;\n+\tint ret = 0;\n+\tstruct mlx5dv_obj obj;\n+\n+\tMLX5_ASSERT(rxq_data);\n+\tMLX5_ASSERT(!rxq_ctrl->obj);\n+\tpriv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_RX_QUEUE;\n+\tpriv->verbs_alloc_ctx.obj = rxq_ctrl;\n+\ttmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,\n+\t\t\t rxq_ctrl->socket);\n+\tif (!tmpl) {\n+\t\tDRV_LOG(ERR, \"port %u Rx queue %u cannot allocate resources\",\n+\t\t\tdev->data->port_id, rxq_data->idx);\n+\t\trte_errno = ENOMEM;\n+\t\tgoto error;\n+\t}\n+\ttmpl->type = MLX5_RXQ_OBJ_TYPE_IBV;\n+\ttmpl->rxq_ctrl = rxq_ctrl;\n+\tif (rxq_ctrl->irq) {\n+\t\ttmpl->ibv_channel =\n+\t\t\t\tmlx5_glue->create_comp_channel(priv->sh->ctx);\n+\t\tif (!tmpl->ibv_channel) {\n+\t\t\tDRV_LOG(ERR, \"Port %u: comp channel creation failure.\",\n+\t\t\t\tdev->data->port_id);\n+\t\t\trte_errno = ENOMEM;\n+\t\t\tgoto error;\n+\t\t}\n+\t\ttmpl->fd = ((struct ibv_comp_channel *)(tmpl->ibv_channel))->fd;\n+\t}\n+\tif (mlx5_rxq_mprq_enabled(rxq_data))\n+\t\tcqe_n = wqe_n * (1 << rxq_data->strd_num_n) - 1;\n+\telse\n+\t\tcqe_n = wqe_n - 1;\n+\tDRV_LOG(DEBUG, \"port %u device_attr.max_qp_wr is %d\",\n+\t\tdev->data->port_id, priv->sh->device_attr.max_qp_wr);\n+\tDRV_LOG(DEBUG, \"port %u device_attr.max_sge is %d\",\n+\t\tdev->data->port_id, priv->sh->device_attr.max_sge);\n+\t/* Create CQ using Verbs API. */\n+\ttmpl->ibv_cq = mlx5_ibv_cq_new(dev, priv, rxq_data, cqe_n, tmpl);\n+\tif (!tmpl->ibv_cq) {\n+\t\tDRV_LOG(ERR, \"Port %u Rx queue %u CQ creation failure.\",\n+\t\t\tdev->data->port_id, idx);\n+\t\trte_errno = ENOMEM;\n+\t\tgoto error;\n+\t}\n+\tobj.cq.in = tmpl->ibv_cq;\n+\tobj.cq.out = &cq_info;\n+\tret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ);\n+\tif (ret) {\n+\t\trte_errno = ret;\n+\t\tgoto error;\n+\t}\n+\tif (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {\n+\t\tDRV_LOG(ERR,\n+\t\t\t\"Port %u wrong MLX5_CQE_SIZE environment \"\n+\t\t\t\"variable value: it should be set to %u.\",\n+\t\t\tdev->data->port_id, RTE_CACHE_LINE_SIZE);\n+\t\trte_errno = EINVAL;\n+\t\tgoto error;\n+\t}\n+\t/* Fill the rings. */\n+\trxq_data->cqe_n = log2above(cq_info.cqe_cnt);\n+\trxq_data->cq_db = cq_info.dbrec;\n+\trxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)cq_info.buf;\n+\trxq_data->cq_uar = cq_info.cq_uar;\n+\trxq_data->cqn = cq_info.cqn;\n+\t/* Create WQ (RQ) using Verbs API. */\n+\ttmpl->wq = mlx5_ibv_wq_new(dev, priv, rxq_data, idx, wqe_n, tmpl);\n+\tif (!tmpl->wq) {\n+\t\tDRV_LOG(ERR, \"Port %u Rx queue %u WQ creation failure.\",\n+\t\t\tdev->data->port_id, idx);\n+\t\trte_errno = ENOMEM;\n+\t\tgoto error;\n+\t}\n+\t/* Change queue state to ready. */\n+\tmod = (struct ibv_wq_attr){\n+\t\t.attr_mask = IBV_WQ_ATTR_STATE,\n+\t\t.wq_state = IBV_WQS_RDY,\n+\t};\n+\tret = mlx5_glue->modify_wq(tmpl->wq, &mod);\n+\tif (ret) {\n+\t\tDRV_LOG(ERR,\n+\t\t\t\"Port %u Rx queue %u WQ state to IBV_WQS_RDY failed.\",\n+\t\t\tdev->data->port_id, idx);\n+\t\trte_errno = ret;\n+\t\tgoto error;\n+\t}\n+\tobj.rwq.in = tmpl->wq;\n+\tobj.rwq.out = &rwq;\n+\tret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_RWQ);\n+\tif (ret) {\n+\t\trte_errno = ret;\n+\t\tgoto error;\n+\t}\n+\trxq_data->wqes = rwq.buf;\n+\trxq_data->rq_db = rwq.dbrec;\n+\trxq_data->cq_arm_sn = 0;\n+\tmlx5_rxq_initialize(rxq_data);\n+\trxq_data->cq_ci = 0;\n+\tDRV_LOG(DEBUG, \"port %u rxq %u updated with %p\", dev->data->port_id,\n+\t\tidx, (void *)&tmpl);\n+\tLIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);\n+\tpriv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;\n+\tdev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;\n+\trxq_ctrl->wqn = ((struct ibv_wq *)(tmpl->wq))->wq_num;\n+\treturn tmpl;\n+error:\n+\tif (tmpl) {\n+\t\tret = rte_errno; /* Save rte_errno before cleanup. */\n+\t\tif (tmpl->wq)\n+\t\t\tclaim_zero(mlx5_glue->destroy_wq(tmpl->wq));\n+\t\tif (tmpl->ibv_cq)\n+\t\t\tclaim_zero(mlx5_glue->destroy_cq(tmpl->ibv_cq));\n+\t\tif (tmpl->ibv_channel)\n+\t\t\tclaim_zero(mlx5_glue->destroy_comp_channel\n+\t\t\t\t\t\t\t(tmpl->ibv_channel));\n+\t\tmlx5_free(tmpl);\n+\t\trte_errno = ret; /* Restore rte_errno. */\n+\t}\n+\tpriv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;\n+\treturn NULL;\n+}\n+\n+/**\n+ * Release an Rx verbs queue object.\n+ *\n+ * @param rxq_obj\n+ * Verbs Rx queue object.\n+ */\n+static void\n+mlx5_rxq_ibv_obj_release(struct mlx5_rxq_obj *rxq_obj)\n+{\n+\tMLX5_ASSERT(rxq_obj);\n+\tMLX5_ASSERT(rxq_obj->wq);\n+\tMLX5_ASSERT(rxq_obj->ibv_cq);\n+\trxq_free_elts(rxq_obj->rxq_ctrl);\n+\tclaim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));\n+\tclaim_zero(mlx5_glue->destroy_cq(rxq_obj->ibv_cq));\n+\tif (rxq_obj->ibv_channel)\n+\t\tclaim_zero(mlx5_glue->destroy_comp_channel\n+\t\t\t\t\t\t\t(rxq_obj->ibv_channel));\n+\tLIST_REMOVE(rxq_obj, next);\n+\tmlx5_free(rxq_obj);\n+}\n+\n struct mlx5_obj_ops ibv_obj_ops = {\n \t.rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_wq_vlan_strip,\n+\t.rxq_obj_new = mlx5_rxq_ibv_obj_new,\n+\t.rxq_obj_release = mlx5_rxq_ibv_obj_release,\n };\ndiff --git a/drivers/net/mlx5/linux/mlx5_verbs.h b/drivers/net/mlx5/linux/mlx5_verbs.h\nindex 4f0b637..2e69c0f 100644\n--- a/drivers/net/mlx5/linux/mlx5_verbs.h\n+++ b/drivers/net/mlx5/linux/mlx5_verbs.h\n@@ -5,6 +5,8 @@\n #ifndef RTE_PMD_MLX5_VERBS_H_\n #define RTE_PMD_MLX5_VERBS_H_\n \n+#include \"mlx5.h\"\n+\n struct mlx5_verbs_ops {\n \tmlx5_reg_mr_t reg_mr;\n \tmlx5_dereg_mr_t dereg_mr;\n@@ -12,4 +14,6 @@ struct mlx5_verbs_ops {\n \n /* Verbs ops struct */\n extern const struct mlx5_verbs_ops mlx5_verbs_ops;\n+extern struct mlx5_obj_ops ibv_obj_ops;\n+\n #endif /* RTE_PMD_MLX5_VERBS_H_ */\ndiff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build\nindex 23462d1..9a97bb9 100644\n--- a/drivers/net/mlx5/meson.build\n+++ b/drivers/net/mlx5/meson.build\n@@ -28,6 +28,7 @@ sources = files(\n \t'mlx5_txpp.c',\n \t'mlx5_vlan.c',\n \t'mlx5_utils.c',\n+\t'mlx5_devx.c',\n )\n if (dpdk_conf.has('RTE_ARCH_X86_64')\n \tor dpdk_conf.has('RTE_ARCH_ARM64')\ndiff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h\nindex f29a12c..eba5df9 100644\n--- a/drivers/net/mlx5/mlx5.h\n+++ b/drivers/net/mlx5/mlx5.h\n@@ -676,9 +676,40 @@ struct mlx5_proc_priv {\n #define MLX5_PROC_PRIV(port_id) \\\n \t((struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private)\n \n+enum mlx5_rxq_obj_type {\n+\tMLX5_RXQ_OBJ_TYPE_IBV, /* mlx5_rxq_obj with ibv_wq. */\n+\tMLX5_RXQ_OBJ_TYPE_DEVX_RQ, /* mlx5_rxq_obj with mlx5_devx_rq. */\n+\tMLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN,\n+\t/* mlx5_rxq_obj with mlx5_devx_rq and hairpin support. */\n+};\n+\n+/* Verbs/DevX Rx queue elements. */\n+struct mlx5_rxq_obj {\n+\tLIST_ENTRY(mlx5_rxq_obj) next; /* Pointer to the next element. */\n+\tstruct mlx5_rxq_ctrl *rxq_ctrl; /* Back pointer to parent. */\n+\tenum mlx5_rxq_obj_type type;\n+\tint fd; /* File descriptor for event channel */\n+\tRTE_STD_C11\n+\tunion {\n+\t\tstruct {\n+\t\t\tvoid *wq; /* Work Queue. */\n+\t\t\tvoid *ibv_cq; /* Completion Queue. */\n+\t\t\tvoid *ibv_channel;\n+\t\t};\n+\t\tstruct {\n+\t\t\tstruct mlx5_devx_obj *rq; /* DevX Rx Queue object. */\n+\t\t\tstruct mlx5_devx_obj *devx_cq; /* DevX CQ object. */\n+\t\t\tvoid *devx_channel;\n+\t\t};\n+\t};\n+};\n+\n /* HW objects operations structure. */\n struct mlx5_obj_ops {\n \tint (*rxq_obj_modify_vlan_strip)(struct mlx5_rxq_obj *rxq_obj, int on);\n+\tstruct mlx5_rxq_obj *(*rxq_obj_new)(struct rte_eth_dev *dev,\n+\t\t\t\t\t uint16_t idx);\n+\tvoid (*rxq_obj_release)(struct mlx5_rxq_obj *rxq_obj);\n };\n \n struct mlx5_priv {\ndiff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c\nindex 7340412..191b3c2 100644\n--- a/drivers/net/mlx5/mlx5_devx.c\n+++ b/drivers/net/mlx5/mlx5_devx.c\n@@ -43,6 +43,568 @@\n \treturn mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr);\n }\n \n+/**\n+ * Release the resources allocated for an RQ DevX object.\n+ *\n+ * @param rxq_ctrl\n+ * DevX Rx queue object.\n+ */\n+static void\n+rxq_release_devx_rq_resources(struct mlx5_rxq_ctrl *rxq_ctrl)\n+{\n+\tif (rxq_ctrl->rxq.wqes) {\n+\t\tmlx5_free((void *)(uintptr_t)rxq_ctrl->rxq.wqes);\n+\t\trxq_ctrl->rxq.wqes = NULL;\n+\t}\n+\tif (rxq_ctrl->wq_umem) {\n+\t\tmlx5_glue->devx_umem_dereg(rxq_ctrl->wq_umem);\n+\t\trxq_ctrl->wq_umem = NULL;\n+\t}\n+}\n+\n+/**\n+ * Release the resources allocated for the Rx CQ DevX object.\n+ *\n+ * @param rxq_ctrl\n+ * DevX Rx queue object.\n+ */\n+static void\n+rxq_release_devx_cq_resources(struct mlx5_rxq_ctrl *rxq_ctrl)\n+{\n+\tif (rxq_ctrl->rxq.cqes) {\n+\t\trte_free((void *)(uintptr_t)rxq_ctrl->rxq.cqes);\n+\t\trxq_ctrl->rxq.cqes = NULL;\n+\t}\n+\tif (rxq_ctrl->cq_umem) {\n+\t\tmlx5_glue->devx_umem_dereg(rxq_ctrl->cq_umem);\n+\t\trxq_ctrl->cq_umem = NULL;\n+\t}\n+}\n+\n+/**\n+ * Release an Rx hairpin related resources.\n+ *\n+ * @param rxq_obj\n+ * Hairpin Rx queue object.\n+ */\n+static void\n+mlx5_rxq_obj_hairpin_release(struct mlx5_rxq_obj *rxq_obj)\n+{\n+\tstruct mlx5_devx_modify_rq_attr rq_attr = { 0 };\n+\n+\tMLX5_ASSERT(rxq_obj);\n+\trq_attr.state = MLX5_RQC_STATE_RST;\n+\trq_attr.rq_state = MLX5_RQC_STATE_RDY;\n+\tmlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr);\n+\tclaim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq));\n+}\n+\n+/**\n+ * Release an Rx DevX queue object.\n+ *\n+ * @param rxq_obj\n+ * DevX Rx queue object.\n+ */\n+static void\n+mlx5_rxq_devx_obj_release(struct mlx5_rxq_obj *rxq_obj)\n+{\n+\tstruct mlx5_priv *priv = rxq_obj->rxq_ctrl->priv;\n+\n+\tMLX5_ASSERT(rxq_obj);\n+\tMLX5_ASSERT(rxq_obj->rq);\n+\tif (rxq_obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN) {\n+\t\tmlx5_rxq_obj_hairpin_release(rxq_obj);\n+\t} else {\n+\t\tMLX5_ASSERT(rxq_obj->devx_cq);\n+\t\trxq_free_elts(rxq_obj->rxq_ctrl);\n+\t\tclaim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq));\n+\t\tclaim_zero(mlx5_devx_cmd_destroy(rxq_obj->devx_cq));\n+\t\tclaim_zero(mlx5_release_dbr(&priv->dbrpgs,\n+\t\t\t\t\t rxq_obj->rxq_ctrl->rq_dbr_umem_id,\n+\t\t\t\t\t rxq_obj->rxq_ctrl->rq_dbr_offset));\n+\t\tclaim_zero(mlx5_release_dbr(&priv->dbrpgs,\n+\t\t\t\t\t rxq_obj->rxq_ctrl->cq_dbr_umem_id,\n+\t\t\t\t\t rxq_obj->rxq_ctrl->cq_dbr_offset));\n+\t\tif (rxq_obj->devx_channel)\n+\t\t\tmlx5_glue->devx_destroy_event_channel\n+\t\t\t\t\t\t\t(rxq_obj->devx_channel);\n+\t\trxq_release_devx_rq_resources(rxq_obj->rxq_ctrl);\n+\t\trxq_release_devx_cq_resources(rxq_obj->rxq_ctrl);\n+\t}\n+\tLIST_REMOVE(rxq_obj, next);\n+\tmlx5_free(rxq_obj);\n+}\n+\n+/**\n+ * Fill common fields of create RQ attributes structure.\n+ *\n+ * @param rxq_data\n+ * Pointer to Rx queue data.\n+ * @param cqn\n+ * CQ number to use with this RQ.\n+ * @param rq_attr\n+ * RQ attributes structure to fill..\n+ */\n+static void\n+mlx5_devx_create_rq_attr_fill(struct mlx5_rxq_data *rxq_data, uint32_t cqn,\n+\t\t\t struct mlx5_devx_create_rq_attr *rq_attr)\n+{\n+\trq_attr->state = MLX5_RQC_STATE_RST;\n+\trq_attr->vsd = (rxq_data->vlan_strip) ? 0 : 1;\n+\trq_attr->cqn = cqn;\n+\trq_attr->scatter_fcs = (rxq_data->crc_present) ? 1 : 0;\n+}\n+\n+/**\n+ * Fill common fields of DevX WQ attributes structure.\n+ *\n+ * @param priv\n+ * Pointer to device private data.\n+ * @param rxq_ctrl\n+ * Pointer to Rx queue control structure.\n+ * @param wq_attr\n+ * WQ attributes structure to fill..\n+ */\n+static void\n+mlx5_devx_wq_attr_fill(struct mlx5_priv *priv, struct mlx5_rxq_ctrl *rxq_ctrl,\n+\t\t struct mlx5_devx_wq_attr *wq_attr)\n+{\n+\twq_attr->end_padding_mode = priv->config.cqe_pad ?\n+\t\t\t\t\tMLX5_WQ_END_PAD_MODE_ALIGN :\n+\t\t\t\t\tMLX5_WQ_END_PAD_MODE_NONE;\n+\twq_attr->pd = priv->sh->pdn;\n+\twq_attr->dbr_addr = rxq_ctrl->rq_dbr_offset;\n+\twq_attr->dbr_umem_id = rxq_ctrl->rq_dbr_umem_id;\n+\twq_attr->dbr_umem_valid = 1;\n+\twq_attr->wq_umem_id = mlx5_os_get_umem_id(rxq_ctrl->wq_umem);\n+\twq_attr->wq_umem_valid = 1;\n+}\n+\n+/**\n+ * Create a RQ object using DevX.\n+ *\n+ * @param dev\n+ * Pointer to Ethernet device.\n+ * @param idx\n+ * Queue index in DPDK Rx queue array.\n+ * @param cqn\n+ * CQ number to use with this RQ.\n+ *\n+ * @return\n+ * The DevX object initialized, NULL otherwise and rte_errno is set.\n+ */\n+static struct mlx5_devx_obj *\n+mlx5_devx_rq_new(struct rte_eth_dev *dev, uint16_t idx, uint32_t cqn)\n+{\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n+\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n+\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n+\tstruct mlx5_devx_create_rq_attr rq_attr = { 0 };\n+\tuint32_t wqe_n = 1 << (rxq_data->elts_n - rxq_data->sges_n);\n+\tuint32_t wq_size = 0;\n+\tuint32_t wqe_size = 0;\n+\tuint32_t log_wqe_size = 0;\n+\tvoid *buf = NULL;\n+\tstruct mlx5_devx_obj *rq;\n+\n+\t/* Fill RQ attributes. */\n+\trq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE;\n+\trq_attr.flush_in_error_en = 1;\n+\tmlx5_devx_create_rq_attr_fill(rxq_data, cqn, &rq_attr);\n+\t/* Fill WQ attributes for this RQ. */\n+\tif (mlx5_rxq_mprq_enabled(rxq_data)) {\n+\t\trq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ;\n+\t\t/*\n+\t\t * Number of strides in each WQE:\n+\t\t * 512*2^single_wqe_log_num_of_strides.\n+\t\t */\n+\t\trq_attr.wq_attr.single_wqe_log_num_of_strides =\n+\t\t\t\trxq_data->strd_num_n -\n+\t\t\t\tMLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;\n+\t\t/* Stride size = (2^single_stride_log_num_of_bytes)*64B. */\n+\t\trq_attr.wq_attr.single_stride_log_num_of_bytes =\n+\t\t\t\trxq_data->strd_sz_n -\n+\t\t\t\tMLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;\n+\t\twqe_size = sizeof(struct mlx5_wqe_mprq);\n+\t} else {\n+\t\trq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;\n+\t\twqe_size = sizeof(struct mlx5_wqe_data_seg);\n+\t}\n+\tlog_wqe_size = log2above(wqe_size) + rxq_data->sges_n;\n+\trq_attr.wq_attr.log_wq_stride = log_wqe_size;\n+\trq_attr.wq_attr.log_wq_sz = rxq_data->elts_n - rxq_data->sges_n;\n+\t/* Calculate and allocate WQ memory space. */\n+\twqe_size = 1 << log_wqe_size; /* round up power of two.*/\n+\twq_size = wqe_n * wqe_size;\n+\tsize_t alignment = MLX5_WQE_BUF_ALIGNMENT;\n+\tif (alignment == (size_t)-1) {\n+\t\tDRV_LOG(ERR, \"Failed to get mem page size\");\n+\t\trte_errno = ENOMEM;\n+\t\treturn NULL;\n+\t}\n+\tbuf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size,\n+\t\t\t alignment, rxq_ctrl->socket);\n+\tif (!buf)\n+\t\treturn NULL;\n+\trxq_data->wqes = buf;\n+\trxq_ctrl->wq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx,\n+\t\t\t\t\t\t buf, wq_size, 0);\n+\tif (!rxq_ctrl->wq_umem) {\n+\t\tmlx5_free(buf);\n+\t\treturn NULL;\n+\t}\n+\tmlx5_devx_wq_attr_fill(priv, rxq_ctrl, &rq_attr.wq_attr);\n+\trq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &rq_attr, rxq_ctrl->socket);\n+\tif (!rq)\n+\t\trxq_release_devx_rq_resources(rxq_ctrl);\n+\treturn rq;\n+}\n+\n+/**\n+ * Create a DevX CQ object for an Rx queue.\n+ *\n+ * @param dev\n+ * Pointer to Ethernet device.\n+ * @param cqe_n\n+ * Number of CQEs in CQ.\n+ * @param idx\n+ * Queue index in DPDK Rx queue array.\n+ * @param rxq_obj\n+ * Pointer to Rx queue object data.\n+ *\n+ * @return\n+ * The DevX object initialized, NULL otherwise and rte_errno is set.\n+ */\n+static struct mlx5_devx_obj *\n+mlx5_devx_cq_new(struct rte_eth_dev *dev, unsigned int cqe_n, uint16_t idx,\n+\t\t struct mlx5_rxq_obj *rxq_obj)\n+{\n+\tstruct mlx5_devx_obj *cq_obj = 0;\n+\tstruct mlx5_devx_cq_attr cq_attr = { 0 };\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n+\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n+\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n+\tsize_t page_size = rte_mem_page_size();\n+\tuint32_t lcore = (uint32_t)rte_lcore_to_cpu_id(-1);\n+\tuint32_t eqn = 0;\n+\tvoid *buf = NULL;\n+\tuint16_t event_nums[1] = {0};\n+\tuint32_t log_cqe_n;\n+\tuint32_t cq_size;\n+\tint ret = 0;\n+\n+\tif (page_size == (size_t)-1) {\n+\t\tDRV_LOG(ERR, \"Failed to get page_size.\");\n+\t\tgoto error;\n+\t}\n+\tif (priv->config.cqe_comp && !rxq_data->hw_timestamp &&\n+\t !rxq_data->lro) {\n+\t\tcq_attr.cqe_comp_en = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;\n+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT\n+\t\tcq_attr.mini_cqe_res_format =\n+\t\t\t\tmlx5_rxq_mprq_enabled(rxq_data) ?\n+\t\t\t\tMLX5DV_CQE_RES_FORMAT_CSUM_STRIDX :\n+\t\t\t\tMLX5DV_CQE_RES_FORMAT_HASH;\n+#else\n+\t\tcq_attr.mini_cqe_res_format = MLX5DV_CQE_RES_FORMAT_HASH;\n+#endif\n+\t\t/*\n+\t\t * For vectorized Rx, it must not be doubled in order to\n+\t\t * make cq_ci and rq_ci aligned.\n+\t\t */\n+\t\tif (mlx5_rxq_check_vec_support(rxq_data) < 0)\n+\t\t\tcqe_n *= 2;\n+\t} else if (priv->config.cqe_comp && rxq_data->hw_timestamp) {\n+\t\tDRV_LOG(DEBUG,\n+\t\t\t\"Port %u Rx CQE compression is disabled for HW\"\n+\t\t\t\" timestamp.\",\n+\t\t\tdev->data->port_id);\n+\t} else if (priv->config.cqe_comp && rxq_data->lro) {\n+\t\tDRV_LOG(DEBUG,\n+\t\t\t\"Port %u Rx CQE compression is disabled for LRO.\",\n+\t\t\tdev->data->port_id);\n+\t}\n+#ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD\n+\tif (priv->config.cqe_pad)\n+\t\tcq_attr.cqe_size = MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD;\n+#endif\n+\tlog_cqe_n = log2above(cqe_n);\n+\tcq_size = sizeof(struct mlx5_cqe) * (1 << log_cqe_n);\n+\t/* Query the EQN for this core. */\n+\tif (mlx5_glue->devx_query_eqn(priv->sh->ctx, lcore, &eqn)) {\n+\t\tDRV_LOG(ERR, \"Failed to query EQN for CQ.\");\n+\t\tgoto error;\n+\t}\n+\tcq_attr.eqn = eqn;\n+\tbuf = rte_calloc_socket(__func__, 1, cq_size, page_size,\n+\t\t\t\trxq_ctrl->socket);\n+\tif (!buf) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate memory for CQ.\");\n+\t\tgoto error;\n+\t}\n+\trxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)buf;\n+\trxq_ctrl->cq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, buf,\n+\t\t\t\t\t\t cq_size,\n+\t\t\t\t\t\t IBV_ACCESS_LOCAL_WRITE);\n+\tif (!rxq_ctrl->cq_umem) {\n+\t\tDRV_LOG(ERR, \"Failed to register umem for CQ.\");\n+\t\tgoto error;\n+\t}\n+\tcq_attr.uar_page_id =\n+\t\t\tmlx5_os_get_devx_uar_page_id(priv->sh->devx_rx_uar);\n+\tcq_attr.q_umem_id = mlx5_os_get_umem_id(rxq_ctrl->cq_umem);\n+\tcq_attr.q_umem_valid = 1;\n+\tcq_attr.log_cq_size = log_cqe_n;\n+\tcq_attr.log_page_size = rte_log2_u32(page_size);\n+\tcq_attr.db_umem_offset = rxq_ctrl->cq_dbr_offset;\n+\tcq_attr.db_umem_id = rxq_ctrl->cq_dbr_umem_id;\n+\tcq_attr.db_umem_valid = 1;\n+\tcq_obj = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr);\n+\tif (!cq_obj)\n+\t\tgoto error;\n+\trxq_data->cqe_n = log_cqe_n;\n+\trxq_data->cqn = cq_obj->id;\n+\tif (rxq_obj->devx_channel) {\n+\t\tret = mlx5_glue->devx_subscribe_devx_event\n+\t\t\t\t\t\t(rxq_obj->devx_channel,\n+\t\t\t\t\t\t cq_obj->obj,\n+\t\t\t\t\t\t sizeof(event_nums),\n+\t\t\t\t\t\t event_nums,\n+\t\t\t\t\t\t (uint64_t)(uintptr_t)cq_obj);\n+\t\tif (ret) {\n+\t\t\tDRV_LOG(ERR, \"Fail to subscribe CQ to event channel.\");\n+\t\t\trte_errno = errno;\n+\t\t\tgoto error;\n+\t\t}\n+\t}\n+\t/* Initialise CQ to 1's to mark HW ownership for all CQEs. */\n+\tmemset((void *)(uintptr_t)rxq_data->cqes, 0xFF, cq_size);\n+\treturn cq_obj;\n+error:\n+\tif (cq_obj)\n+\t\tmlx5_devx_cmd_destroy(cq_obj);\n+\trxq_release_devx_cq_resources(rxq_ctrl);\n+\treturn NULL;\n+}\n+\n+/**\n+ * Create the Rx hairpin queue object.\n+ *\n+ * @param dev\n+ * Pointer to Ethernet device.\n+ * @param idx\n+ * Queue index in DPDK Rx queue array.\n+ *\n+ * @return\n+ * The hairpin DevX object initialized, NULL otherwise and rte_errno is set.\n+ */\n+static struct mlx5_rxq_obj *\n+mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx)\n+{\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n+\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n+\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n+\tstruct mlx5_devx_create_rq_attr attr = { 0 };\n+\tstruct mlx5_rxq_obj *tmpl = NULL;\n+\tuint32_t max_wq_data;\n+\n+\tMLX5_ASSERT(rxq_data);\n+\tMLX5_ASSERT(!rxq_ctrl->obj);\n+\ttmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,\n+\t\t\t rxq_ctrl->socket);\n+\tif (!tmpl) {\n+\t\tDRV_LOG(ERR, \"port %u Rx queue %u cannot allocate resources\",\n+\t\t\tdev->data->port_id, rxq_data->idx);\n+\t\trte_errno = ENOMEM;\n+\t\treturn NULL;\n+\t}\n+\ttmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN;\n+\ttmpl->rxq_ctrl = rxq_ctrl;\n+\tattr.hairpin = 1;\n+\tmax_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz;\n+\t/* Jumbo frames > 9KB should be supported, and more packets. */\n+\tif (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) {\n+\t\tif (priv->config.log_hp_size > max_wq_data) {\n+\t\t\tDRV_LOG(ERR, \"Total data size %u power of 2 is \"\n+\t\t\t\t\"too large for hairpin.\",\n+\t\t\t\tpriv->config.log_hp_size);\n+\t\t\tmlx5_free(tmpl);\n+\t\t\trte_errno = ERANGE;\n+\t\t\treturn NULL;\n+\t\t}\n+\t\tattr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size;\n+\t} else {\n+\t\tattr.wq_attr.log_hairpin_data_sz =\n+\t\t\t\t(max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ?\n+\t\t\t\t max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE;\n+\t}\n+\t/* Set the packets number to the maximum value for performance. */\n+\tattr.wq_attr.log_hairpin_num_packets =\n+\t\t\tattr.wq_attr.log_hairpin_data_sz -\n+\t\t\tMLX5_HAIRPIN_QUEUE_STRIDE;\n+\ttmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &attr,\n+\t\t\t\t\t rxq_ctrl->socket);\n+\tif (!tmpl->rq) {\n+\t\tDRV_LOG(ERR,\n+\t\t\t\"Port %u Rx hairpin queue %u can't create rq object.\",\n+\t\t\tdev->data->port_id, idx);\n+\t\tmlx5_free(tmpl);\n+\t\trte_errno = errno;\n+\t\treturn NULL;\n+\t}\n+\tDRV_LOG(DEBUG, \"port %u rxq %u updated with %p\", dev->data->port_id,\n+\t\tidx, (void *)&tmpl);\n+\tLIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);\n+\tdev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN;\n+\treturn tmpl;\n+}\n+\n+/**\n+ * Create the Rx queue DevX object.\n+ *\n+ * @param dev\n+ * Pointer to Ethernet device.\n+ * @param idx\n+ * Queue index in DPDK Rx queue array.\n+ *\n+ * @return\n+ * The DevX object initialized, NULL otherwise and rte_errno is set.\n+ */\n+static struct mlx5_rxq_obj *\n+mlx5_rxq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx)\n+{\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n+\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n+\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n+\tunsigned int cqe_n;\n+\tunsigned int wqe_n = 1 << rxq_data->elts_n;\n+\tstruct mlx5_rxq_obj *tmpl = NULL;\n+\tstruct mlx5_devx_modify_rq_attr rq_attr = { 0 };\n+\tstruct mlx5_devx_dbr_page *cq_dbr_page = NULL;\n+\tstruct mlx5_devx_dbr_page *rq_dbr_page = NULL;\n+\tint64_t dbr_offset;\n+\tint ret = 0;\n+\n+\tMLX5_ASSERT(rxq_data);\n+\tMLX5_ASSERT(!rxq_ctrl->obj);\n+\tif (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN)\n+\t\treturn mlx5_rxq_obj_hairpin_new(dev, idx);\n+\ttmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,\n+\t\t\t rxq_ctrl->socket);\n+\tif (!tmpl) {\n+\t\tDRV_LOG(ERR, \"port %u Rx queue %u cannot allocate resources\",\n+\t\t\tdev->data->port_id, rxq_data->idx);\n+\t\trte_errno = ENOMEM;\n+\t\tgoto error;\n+\t}\n+\ttmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ;\n+\ttmpl->rxq_ctrl = rxq_ctrl;\n+\tif (rxq_ctrl->irq) {\n+\t\tint devx_ev_flag =\n+\t\t\t MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA;\n+\n+\t\ttmpl->devx_channel = mlx5_glue->devx_create_event_channel\n+\t\t\t\t\t\t\t\t(priv->sh->ctx,\n+\t\t\t\t\t\t\t\t devx_ev_flag);\n+\t\tif (!tmpl->devx_channel) {\n+\t\t\trte_errno = errno;\n+\t\t\tDRV_LOG(ERR, \"Failed to create event channel %d.\",\n+\t\t\t\trte_errno);\n+\t\t\tgoto error;\n+\t\t}\n+\t\ttmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel);\n+\t}\n+\tif (mlx5_rxq_mprq_enabled(rxq_data))\n+\t\tcqe_n = wqe_n * (1 << rxq_data->strd_num_n) - 1;\n+\telse\n+\t\tcqe_n = wqe_n - 1;\n+\tDRV_LOG(DEBUG, \"port %u device_attr.max_qp_wr is %d\",\n+\t\tdev->data->port_id, priv->sh->device_attr.max_qp_wr);\n+\tDRV_LOG(DEBUG, \"port %u device_attr.max_sge is %d\",\n+\t\tdev->data->port_id, priv->sh->device_attr.max_sge);\n+\t/* Allocate CQ door-bell. */\n+\tdbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &cq_dbr_page);\n+\tif (dbr_offset < 0) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate CQ door-bell.\");\n+\t\tgoto error;\n+\t}\n+\trxq_ctrl->cq_dbr_offset = dbr_offset;\n+\trxq_ctrl->cq_dbr_umem_id = mlx5_os_get_umem_id(cq_dbr_page->umem);\n+\trxq_data->cq_db = (uint32_t *)((uintptr_t)cq_dbr_page->dbrs +\n+\t\t\t\t (uintptr_t)rxq_ctrl->cq_dbr_offset);\n+\trxq_data->cq_uar =\n+\t\t\tmlx5_os_get_devx_uar_base_addr(priv->sh->devx_rx_uar);\n+\t/* Create CQ using DevX API. */\n+\ttmpl->devx_cq = mlx5_devx_cq_new(dev, cqe_n, idx, tmpl);\n+\tif (!tmpl->devx_cq) {\n+\t\tDRV_LOG(ERR, \"Failed to create CQ.\");\n+\t\tgoto error;\n+\t}\n+\t/* Allocate RQ door-bell. */\n+\tdbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &rq_dbr_page);\n+\tif (dbr_offset < 0) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate RQ door-bell.\");\n+\t\tgoto error;\n+\t}\n+\trxq_ctrl->rq_dbr_offset = dbr_offset;\n+\trxq_ctrl->rq_dbr_umem_id = mlx5_os_get_umem_id(rq_dbr_page->umem);\n+\trxq_data->rq_db = (uint32_t *)((uintptr_t)rq_dbr_page->dbrs +\n+\t\t\t\t (uintptr_t)rxq_ctrl->rq_dbr_offset);\n+\t/* Create RQ using DevX API. */\n+\ttmpl->rq = mlx5_devx_rq_new(dev, idx, tmpl->devx_cq->id);\n+\tif (!tmpl->rq) {\n+\t\tDRV_LOG(ERR, \"Port %u Rx queue %u RQ creation failure.\",\n+\t\t\tdev->data->port_id, idx);\n+\t\trte_errno = ENOMEM;\n+\t\tgoto error;\n+\t}\n+\t/* Change queue state to ready. */\n+\trq_attr.rq_state = MLX5_RQC_STATE_RST;\n+\trq_attr.state = MLX5_RQC_STATE_RDY;\n+\tret = mlx5_devx_cmd_modify_rq(tmpl->rq, &rq_attr);\n+\tif (ret)\n+\t\tgoto error;\n+\trxq_data->cq_arm_sn = 0;\n+\tmlx5_rxq_initialize(rxq_data);\n+\trxq_data->cq_ci = 0;\n+\tDRV_LOG(DEBUG, \"port %u rxq %u updated with %p\", dev->data->port_id,\n+\t\tidx, (void *)&tmpl);\n+\tLIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);\n+\tdev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;\n+\trxq_ctrl->wqn = tmpl->rq->id;\n+\treturn tmpl;\n+error:\n+\tif (tmpl) {\n+\t\tret = rte_errno; /* Save rte_errno before cleanup. */\n+\t\tif (tmpl->rq)\n+\t\t\tclaim_zero(mlx5_devx_cmd_destroy(tmpl->rq));\n+\t\tif (tmpl->devx_cq)\n+\t\t\tclaim_zero(mlx5_devx_cmd_destroy(tmpl->devx_cq));\n+\t\tif (tmpl->devx_channel)\n+\t\t\tmlx5_glue->devx_destroy_event_channel\n+\t\t\t\t\t\t\t(tmpl->devx_channel);\n+\t\tmlx5_free(tmpl);\n+\t\trte_errno = ret; /* Restore rte_errno. */\n+\t}\n+\tif (rq_dbr_page)\n+\t\tclaim_zero(mlx5_release_dbr(&priv->dbrpgs,\n+\t\t\t\t\t rxq_ctrl->rq_dbr_umem_id,\n+\t\t\t\t\t rxq_ctrl->rq_dbr_offset));\n+\tif (cq_dbr_page)\n+\t\tclaim_zero(mlx5_release_dbr(&priv->dbrpgs,\n+\t\t\t\t\t rxq_ctrl->cq_dbr_umem_id,\n+\t\t\t\t\t rxq_ctrl->cq_dbr_offset));\n+\trxq_release_devx_rq_resources(rxq_ctrl);\n+\trxq_release_devx_cq_resources(rxq_ctrl);\n+\treturn NULL;\n+}\n+\n struct mlx5_obj_ops devx_obj_ops = {\n \t.rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip,\n+\t.rxq_obj_new = mlx5_rxq_devx_obj_new,\n+\t.rxq_obj_release = mlx5_rxq_devx_obj_release,\n };\ndiff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c\nindex 506c4d3..daa92b6 100644\n--- a/drivers/net/mlx5/mlx5_rxq.c\n+++ b/drivers/net/mlx5/mlx5_rxq.c\n@@ -347,7 +347,7 @@\n * @param rxq_ctrl\n * Pointer to RX queue structure.\n */\n-static void\n+void\n rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl)\n {\n \tif (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq))\n@@ -832,113 +832,6 @@\n }\n \n /**\n- * Release the resources allocated for an RQ DevX object.\n- *\n- * @param rxq_ctrl\n- * DevX Rx queue object.\n- */\n-static void\n-rxq_release_devx_rq_resources(struct mlx5_rxq_ctrl *rxq_ctrl)\n-{\n-\tif (rxq_ctrl->rxq.wqes) {\n-\t\tmlx5_free((void *)(uintptr_t)rxq_ctrl->rxq.wqes);\n-\t\trxq_ctrl->rxq.wqes = NULL;\n-\t}\n-\tif (rxq_ctrl->wq_umem) {\n-\t\tmlx5_glue->devx_umem_dereg(rxq_ctrl->wq_umem);\n-\t\trxq_ctrl->wq_umem = NULL;\n-\t}\n-}\n-\n-/**\n- * Release the resources allocated for the Rx CQ DevX object.\n- *\n- * @param rxq_ctrl\n- * DevX Rx queue object.\n- */\n-static void\n-rxq_release_devx_cq_resources(struct mlx5_rxq_ctrl *rxq_ctrl)\n-{\n-\tif (rxq_ctrl->rxq.cqes) {\n-\t\trte_free((void *)(uintptr_t)rxq_ctrl->rxq.cqes);\n-\t\trxq_ctrl->rxq.cqes = NULL;\n-\t}\n-\tif (rxq_ctrl->cq_umem) {\n-\t\tmlx5_glue->devx_umem_dereg(rxq_ctrl->cq_umem);\n-\t\trxq_ctrl->cq_umem = NULL;\n-\t}\n-}\n-\n-/**\n- * Release an Rx hairpin related resources.\n- *\n- * @param rxq_obj\n- * Hairpin Rx queue object.\n- */\n-static void\n-rxq_obj_hairpin_release(struct mlx5_rxq_obj *rxq_obj)\n-{\n-\tstruct mlx5_devx_modify_rq_attr rq_attr = { 0 };\n-\n-\tMLX5_ASSERT(rxq_obj);\n-\trq_attr.state = MLX5_RQC_STATE_RST;\n-\trq_attr.rq_state = MLX5_RQC_STATE_RDY;\n-\tmlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr);\n-\tclaim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq));\n-}\n-\n-/**\n- * Release an Rx verbs/DevX queue object.\n- *\n- * @param rxq_obj\n- * Verbs/DevX Rx queue object.\n- */\n-static void\n-mlx5_rxq_obj_release(struct mlx5_rxq_obj *rxq_obj)\n-{\n-\tstruct mlx5_priv *priv = rxq_obj->rxq_ctrl->priv;\n-\tstruct mlx5_rxq_ctrl *rxq_ctrl = rxq_obj->rxq_ctrl;\n-\n-\tMLX5_ASSERT(rxq_obj);\n-\tswitch (rxq_obj->type) {\n-\tcase MLX5_RXQ_OBJ_TYPE_IBV:\n-\t\tMLX5_ASSERT(rxq_obj->wq);\n-\t\tMLX5_ASSERT(rxq_obj->ibv_cq);\n-\t\trxq_free_elts(rxq_ctrl);\n-\t\tclaim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));\n-\t\tclaim_zero(mlx5_glue->destroy_cq(rxq_obj->ibv_cq));\n-\t\tif (rxq_obj->ibv_channel)\n-\t\t\tclaim_zero(mlx5_glue->destroy_comp_channel\n-\t\t\t\t\t\t\t(rxq_obj->ibv_channel));\n-\t\tbreak;\n-\tcase MLX5_RXQ_OBJ_TYPE_DEVX_RQ:\n-\t\tMLX5_ASSERT(rxq_obj->rq);\n-\t\tMLX5_ASSERT(rxq_obj->devx_cq);\n-\t\trxq_free_elts(rxq_ctrl);\n-\t\tclaim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq));\n-\t\tclaim_zero(mlx5_devx_cmd_destroy(rxq_obj->devx_cq));\n-\t\tclaim_zero(mlx5_release_dbr(&priv->dbrpgs,\n-\t\t\t\t\t rxq_ctrl->rq_dbr_umem_id,\n-\t\t\t\t\t rxq_ctrl->rq_dbr_offset));\n-\t\tclaim_zero(mlx5_release_dbr(&priv->dbrpgs,\n-\t\t\t\t\t rxq_ctrl->cq_dbr_umem_id,\n-\t\t\t\t\t rxq_ctrl->cq_dbr_offset));\n-\t\tif (rxq_obj->devx_channel)\n-\t\t\tmlx5_glue->devx_destroy_event_channel\n-\t\t\t\t\t\t\t(rxq_obj->devx_channel);\n-\t\trxq_release_devx_rq_resources(rxq_ctrl);\n-\t\trxq_release_devx_cq_resources(rxq_ctrl);\n-\t\tbreak;\n-\tcase MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN:\n-\t\tMLX5_ASSERT(rxq_obj->rq);\n-\t\trxq_obj_hairpin_release(rxq_obj);\n-\t\tbreak;\n-\t}\n-\tLIST_REMOVE(rxq_obj, next);\n-\tmlx5_free(rxq_obj);\n-}\n-\n-/**\n * Allocate queue vector and fill epoll fd list for Rx interrupts.\n *\n * @param dev\n@@ -1193,756 +1086,6 @@\n }\n \n /**\n- * Create a CQ Verbs object.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param priv\n- * Pointer to device private data.\n- * @param rxq_data\n- * Pointer to Rx queue data.\n- * @param cqe_n\n- * Number of CQEs in CQ.\n- * @param rxq_obj\n- * Pointer to Rx queue object data.\n- *\n- * @return\n- * The Verbs object initialised, NULL otherwise and rte_errno is set.\n- */\n-static struct ibv_cq *\n-mlx5_ibv_cq_new(struct rte_eth_dev *dev, struct mlx5_priv *priv,\n-\t\tstruct mlx5_rxq_data *rxq_data,\n-\t\tunsigned int cqe_n, struct mlx5_rxq_obj *rxq_obj)\n-{\n-\tstruct {\n-\t\tstruct ibv_cq_init_attr_ex ibv;\n-\t\tstruct mlx5dv_cq_init_attr mlx5;\n-\t} cq_attr;\n-\n-\tcq_attr.ibv = (struct ibv_cq_init_attr_ex){\n-\t\t.cqe = cqe_n,\n-\t\t.channel = rxq_obj->ibv_channel,\n-\t\t.comp_mask = 0,\n-\t};\n-\tcq_attr.mlx5 = (struct mlx5dv_cq_init_attr){\n-\t\t.comp_mask = 0,\n-\t};\n-\tif (priv->config.cqe_comp && !rxq_data->hw_timestamp) {\n-\t\tcq_attr.mlx5.comp_mask |=\n-\t\t\t\tMLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;\n-#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT\n-\t\tcq_attr.mlx5.cqe_comp_res_format =\n-\t\t\t\tmlx5_rxq_mprq_enabled(rxq_data) ?\n-\t\t\t\tMLX5DV_CQE_RES_FORMAT_CSUM_STRIDX :\n-\t\t\t\tMLX5DV_CQE_RES_FORMAT_HASH;\n-#else\n-\t\tcq_attr.mlx5.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;\n-#endif\n-\t\t/*\n-\t\t * For vectorized Rx, it must not be doubled in order to\n-\t\t * make cq_ci and rq_ci aligned.\n-\t\t */\n-\t\tif (mlx5_rxq_check_vec_support(rxq_data) < 0)\n-\t\t\tcq_attr.ibv.cqe *= 2;\n-\t} else if (priv->config.cqe_comp && rxq_data->hw_timestamp) {\n-\t\tDRV_LOG(DEBUG,\n-\t\t\t\"port %u Rx CQE compression is disabled for HW\"\n-\t\t\t\" timestamp\",\n-\t\t\tdev->data->port_id);\n-\t}\n-#ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD\n-\tif (priv->config.cqe_pad) {\n-\t\tcq_attr.mlx5.comp_mask |= MLX5DV_CQ_INIT_ATTR_MASK_FLAGS;\n-\t\tcq_attr.mlx5.flags |= MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD;\n-\t}\n-#endif\n-\treturn mlx5_glue->cq_ex_to_cq(mlx5_glue->dv_create_cq(priv->sh->ctx,\n-\t\t\t\t\t\t\t &cq_attr.ibv,\n-\t\t\t\t\t\t\t &cq_attr.mlx5));\n-}\n-\n-/**\n- * Create a WQ Verbs object.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param priv\n- * Pointer to device private data.\n- * @param rxq_data\n- * Pointer to Rx queue data.\n- * @param idx\n- * Queue index in DPDK Rx queue array\n- * @param wqe_n\n- * Number of WQEs in WQ.\n- * @param rxq_obj\n- * Pointer to Rx queue object data.\n- *\n- * @return\n- * The Verbs object initialised, NULL otherwise and rte_errno is set.\n- */\n-static struct ibv_wq *\n-mlx5_ibv_wq_new(struct rte_eth_dev *dev, struct mlx5_priv *priv,\n-\t\tstruct mlx5_rxq_data *rxq_data, uint16_t idx,\n-\t\tunsigned int wqe_n, struct mlx5_rxq_obj *rxq_obj)\n-{\n-\tstruct {\n-\t\tstruct ibv_wq_init_attr ibv;\n-#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT\n-\t\tstruct mlx5dv_wq_init_attr mlx5;\n-#endif\n-\t} wq_attr;\n-\n-\twq_attr.ibv = (struct ibv_wq_init_attr){\n-\t\t.wq_context = NULL, /* Could be useful in the future. */\n-\t\t.wq_type = IBV_WQT_RQ,\n-\t\t/* Max number of outstanding WRs. */\n-\t\t.max_wr = wqe_n >> rxq_data->sges_n,\n-\t\t/* Max number of scatter/gather elements in a WR. */\n-\t\t.max_sge = 1 << rxq_data->sges_n,\n-\t\t.pd = priv->sh->pd,\n-\t\t.cq = rxq_obj->ibv_cq,\n-\t\t.comp_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING | 0,\n-\t\t.create_flags = (rxq_data->vlan_strip ?\n-\t\t\t\t IBV_WQ_FLAGS_CVLAN_STRIPPING : 0),\n-\t};\n-\t/* By default, FCS (CRC) is stripped by hardware. */\n-\tif (rxq_data->crc_present) {\n-\t\twq_attr.ibv.create_flags |= IBV_WQ_FLAGS_SCATTER_FCS;\n-\t\twq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;\n-\t}\n-\tif (priv->config.hw_padding) {\n-#if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)\n-\t\twq_attr.ibv.create_flags |= IBV_WQ_FLAG_RX_END_PADDING;\n-\t\twq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;\n-#elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)\n-\t\twq_attr.ibv.create_flags |= IBV_WQ_FLAGS_PCI_WRITE_END_PADDING;\n-\t\twq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;\n-#endif\n-\t}\n-#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT\n-\twq_attr.mlx5 = (struct mlx5dv_wq_init_attr){\n-\t\t.comp_mask = 0,\n-\t};\n-\tif (mlx5_rxq_mprq_enabled(rxq_data)) {\n-\t\tstruct mlx5dv_striding_rq_init_attr *mprq_attr =\n-\t\t\t\t\t\t&wq_attr.mlx5.striding_rq_attrs;\n-\n-\t\twq_attr.mlx5.comp_mask |= MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ;\n-\t\t*mprq_attr = (struct mlx5dv_striding_rq_init_attr){\n-\t\t\t.single_stride_log_num_of_bytes = rxq_data->strd_sz_n,\n-\t\t\t.single_wqe_log_num_of_strides = rxq_data->strd_num_n,\n-\t\t\t.two_byte_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT,\n-\t\t};\n-\t}\n-\trxq_obj->wq = mlx5_glue->dv_create_wq(priv->sh->ctx, &wq_attr.ibv,\n-\t\t\t\t\t &wq_attr.mlx5);\n-#else\n-\trxq_obj->wq = mlx5_glue->create_wq(priv->sh->ctx, &wq_attr.ibv);\n-#endif\n-\tif (rxq_obj->wq) {\n-\t\t/*\n-\t\t * Make sure number of WRs*SGEs match expectations since a queue\n-\t\t * cannot allocate more than \"desc\" buffers.\n-\t\t */\n-\t\tif (wq_attr.ibv.max_wr != (wqe_n >> rxq_data->sges_n) ||\n-\t\t wq_attr.ibv.max_sge != (1u << rxq_data->sges_n)) {\n-\t\t\tDRV_LOG(ERR,\n-\t\t\t\t\"port %u Rx queue %u requested %u*%u but got\"\n-\t\t\t\t\" %u*%u WRs*SGEs\",\n-\t\t\t\tdev->data->port_id, idx,\n-\t\t\t\twqe_n >> rxq_data->sges_n,\n-\t\t\t\t(1 << rxq_data->sges_n),\n-\t\t\t\twq_attr.ibv.max_wr, wq_attr.ibv.max_sge);\n-\t\t\tclaim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));\n-\t\t\trxq_obj->wq = NULL;\n-\t\t\trte_errno = EINVAL;\n-\t\t}\n-\t}\n-\treturn rxq_obj->wq;\n-}\n-\n-/**\n- * Fill common fields of create RQ attributes structure.\n- *\n- * @param rxq_data\n- * Pointer to Rx queue data.\n- * @param cqn\n- * CQ number to use with this RQ.\n- * @param rq_attr\n- * RQ attributes structure to fill..\n- */\n-static void\n-mlx5_devx_create_rq_attr_fill(struct mlx5_rxq_data *rxq_data, uint32_t cqn,\n-\t\t\t struct mlx5_devx_create_rq_attr *rq_attr)\n-{\n-\trq_attr->state = MLX5_RQC_STATE_RST;\n-\trq_attr->vsd = (rxq_data->vlan_strip) ? 0 : 1;\n-\trq_attr->cqn = cqn;\n-\trq_attr->scatter_fcs = (rxq_data->crc_present) ? 1 : 0;\n-}\n-\n-/**\n- * Fill common fields of DevX WQ attributes structure.\n- *\n- * @param priv\n- * Pointer to device private data.\n- * @param rxq_ctrl\n- * Pointer to Rx queue control structure.\n- * @param wq_attr\n- * WQ attributes structure to fill..\n- */\n-static void\n-mlx5_devx_wq_attr_fill(struct mlx5_priv *priv, struct mlx5_rxq_ctrl *rxq_ctrl,\n-\t\t struct mlx5_devx_wq_attr *wq_attr)\n-{\n-\twq_attr->end_padding_mode = priv->config.cqe_pad ?\n-\t\t\t\t\tMLX5_WQ_END_PAD_MODE_ALIGN :\n-\t\t\t\t\tMLX5_WQ_END_PAD_MODE_NONE;\n-\twq_attr->pd = priv->sh->pdn;\n-\twq_attr->dbr_addr = rxq_ctrl->rq_dbr_offset;\n-\twq_attr->dbr_umem_id = rxq_ctrl->rq_dbr_umem_id;\n-\twq_attr->dbr_umem_valid = 1;\n-\twq_attr->wq_umem_id = mlx5_os_get_umem_id(rxq_ctrl->wq_umem);\n-\twq_attr->wq_umem_valid = 1;\n-}\n-\n-/**\n- * Create a RQ object using DevX.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param idx\n- * Queue index in DPDK Rx queue array\n- * @param cqn\n- * CQ number to use with this RQ.\n- *\n- * @return\n- * The DevX object initialised, NULL otherwise and rte_errno is set.\n- */\n-static struct mlx5_devx_obj *\n-mlx5_devx_rq_new(struct rte_eth_dev *dev, uint16_t idx, uint32_t cqn)\n-{\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n-\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n-\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n-\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n-\tstruct mlx5_devx_create_rq_attr rq_attr = { 0 };\n-\tuint32_t wqe_n = 1 << (rxq_data->elts_n - rxq_data->sges_n);\n-\tuint32_t wq_size = 0;\n-\tuint32_t wqe_size = 0;\n-\tuint32_t log_wqe_size = 0;\n-\tvoid *buf = NULL;\n-\tstruct mlx5_devx_obj *rq;\n-\n-\t/* Fill RQ attributes. */\n-\trq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE;\n-\trq_attr.flush_in_error_en = 1;\n-\tmlx5_devx_create_rq_attr_fill(rxq_data, cqn, &rq_attr);\n-\t/* Fill WQ attributes for this RQ. */\n-\tif (mlx5_rxq_mprq_enabled(rxq_data)) {\n-\t\trq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ;\n-\t\t/*\n-\t\t * Number of strides in each WQE:\n-\t\t * 512*2^single_wqe_log_num_of_strides.\n-\t\t */\n-\t\trq_attr.wq_attr.single_wqe_log_num_of_strides =\n-\t\t\t\trxq_data->strd_num_n -\n-\t\t\t\tMLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;\n-\t\t/* Stride size = (2^single_stride_log_num_of_bytes)*64B. */\n-\t\trq_attr.wq_attr.single_stride_log_num_of_bytes =\n-\t\t\t\trxq_data->strd_sz_n -\n-\t\t\t\tMLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;\n-\t\twqe_size = sizeof(struct mlx5_wqe_mprq);\n-\t} else {\n-\t\trq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;\n-\t\twqe_size = sizeof(struct mlx5_wqe_data_seg);\n-\t}\n-\tlog_wqe_size = log2above(wqe_size) + rxq_data->sges_n;\n-\trq_attr.wq_attr.log_wq_stride = log_wqe_size;\n-\trq_attr.wq_attr.log_wq_sz = rxq_data->elts_n - rxq_data->sges_n;\n-\t/* Calculate and allocate WQ memory space. */\n-\twqe_size = 1 << log_wqe_size; /* round up power of two.*/\n-\twq_size = wqe_n * wqe_size;\n-\tsize_t alignment = MLX5_WQE_BUF_ALIGNMENT;\n-\tif (alignment == (size_t)-1) {\n-\t\tDRV_LOG(ERR, \"Failed to get mem page size\");\n-\t\trte_errno = ENOMEM;\n-\t\treturn NULL;\n-\t}\n-\tbuf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size,\n-\t\t\t alignment, rxq_ctrl->socket);\n-\tif (!buf)\n-\t\treturn NULL;\n-\trxq_data->wqes = buf;\n-\trxq_ctrl->wq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx,\n-\t\t\t\t\t\t buf, wq_size, 0);\n-\tif (!rxq_ctrl->wq_umem) {\n-\t\tmlx5_free(buf);\n-\t\treturn NULL;\n-\t}\n-\tmlx5_devx_wq_attr_fill(priv, rxq_ctrl, &rq_attr.wq_attr);\n-\trq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &rq_attr, rxq_ctrl->socket);\n-\tif (!rq)\n-\t\trxq_release_devx_rq_resources(rxq_ctrl);\n-\treturn rq;\n-}\n-\n-/**\n- * Create a DevX CQ object for an Rx queue.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param cqe_n\n- * Number of CQEs in CQ.\n- * @param idx\n- * Queue index in DPDK Rx queue array\n- * @param rxq_obj\n- * Pointer to Rx queue object data.\n- *\n- * @return\n- * The DevX object initialised, NULL otherwise and rte_errno is set.\n- */\n-static struct mlx5_devx_obj *\n-mlx5_devx_cq_new(struct rte_eth_dev *dev, unsigned int cqe_n, uint16_t idx,\n-\t\t struct mlx5_rxq_obj *rxq_obj)\n-{\n-\tstruct mlx5_devx_obj *cq_obj = 0;\n-\tstruct mlx5_devx_cq_attr cq_attr = { 0 };\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n-\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n-\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n-\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n-\tsize_t page_size = rte_mem_page_size();\n-\tuint32_t lcore = (uint32_t)rte_lcore_to_cpu_id(-1);\n-\tuint32_t eqn = 0;\n-\tvoid *buf = NULL;\n-\tuint16_t event_nums[1] = {0};\n-\tuint32_t log_cqe_n;\n-\tuint32_t cq_size;\n-\tint ret = 0;\n-\n-\tif (page_size == (size_t)-1) {\n-\t\tDRV_LOG(ERR, \"Failed to get page_size.\");\n-\t\tgoto error;\n-\t}\n-\tif (priv->config.cqe_comp && !rxq_data->hw_timestamp &&\n-\t !rxq_data->lro) {\n-\t\tcq_attr.cqe_comp_en = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;\n-#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT\n-\t\tcq_attr.mini_cqe_res_format =\n-\t\t\t\tmlx5_rxq_mprq_enabled(rxq_data) ?\n-\t\t\t\tMLX5DV_CQE_RES_FORMAT_CSUM_STRIDX :\n-\t\t\t\tMLX5DV_CQE_RES_FORMAT_HASH;\n-#else\n-\t\tcq_attr.mini_cqe_res_format = MLX5DV_CQE_RES_FORMAT_HASH;\n-#endif\n-\t\t/*\n-\t\t * For vectorized Rx, it must not be doubled in order to\n-\t\t * make cq_ci and rq_ci aligned.\n-\t\t */\n-\t\tif (mlx5_rxq_check_vec_support(rxq_data) < 0)\n-\t\t\tcqe_n *= 2;\n-\t} else if (priv->config.cqe_comp && rxq_data->hw_timestamp) {\n-\t\tDRV_LOG(DEBUG,\n-\t\t\t\"port %u Rx CQE compression is disabled for HW\"\n-\t\t\t\" timestamp\",\n-\t\t\tdev->data->port_id);\n-\t} else if (priv->config.cqe_comp && rxq_data->lro) {\n-\t\tDRV_LOG(DEBUG,\n-\t\t\t\"port %u Rx CQE compression is disabled for LRO\",\n-\t\t\tdev->data->port_id);\n-\t}\n-#ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD\n-\tif (priv->config.cqe_pad)\n-\t\tcq_attr.cqe_size = MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD;\n-#endif\n-\tlog_cqe_n = log2above(cqe_n);\n-\tcq_size = sizeof(struct mlx5_cqe) * (1 << log_cqe_n);\n-\t/* Query the EQN for this core. */\n-\tif (mlx5_glue->devx_query_eqn(priv->sh->ctx, lcore, &eqn)) {\n-\t\tDRV_LOG(ERR, \"Failed to query EQN for CQ.\");\n-\t\tgoto error;\n-\t}\n-\tcq_attr.eqn = eqn;\n-\tbuf = rte_calloc_socket(__func__, 1, cq_size, page_size,\n-\t\t\t\trxq_ctrl->socket);\n-\tif (!buf) {\n-\t\tDRV_LOG(ERR, \"Failed to allocate memory for CQ.\");\n-\t\tgoto error;\n-\t}\n-\trxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)buf;\n-\trxq_ctrl->cq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, buf,\n-\t\t\t\t\t\t cq_size,\n-\t\t\t\t\t\t IBV_ACCESS_LOCAL_WRITE);\n-\tif (!rxq_ctrl->cq_umem) {\n-\t\tDRV_LOG(ERR, \"Failed to register umem for CQ.\");\n-\t\tgoto error;\n-\t}\n-\tcq_attr.uar_page_id =\n-\t\tmlx5_os_get_devx_uar_page_id(priv->sh->devx_rx_uar);\n-\tcq_attr.q_umem_id = mlx5_os_get_umem_id(rxq_ctrl->cq_umem);\n-\tcq_attr.q_umem_valid = 1;\n-\tcq_attr.log_cq_size = log_cqe_n;\n-\tcq_attr.log_page_size = rte_log2_u32(page_size);\n-\tcq_attr.db_umem_offset = rxq_ctrl->cq_dbr_offset;\n-\tcq_attr.db_umem_id = rxq_ctrl->cq_dbr_umem_id;\n-\tcq_attr.db_umem_valid = 1;\n-\tcq_obj = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr);\n-\tif (!cq_obj)\n-\t\tgoto error;\n-\trxq_data->cqe_n = log_cqe_n;\n-\trxq_data->cqn = cq_obj->id;\n-\tif (rxq_obj->devx_channel) {\n-\t\tret = mlx5_glue->devx_subscribe_devx_event\n-\t\t\t\t\t\t(rxq_obj->devx_channel,\n-\t\t\t\t\t\t cq_obj->obj,\n-\t\t\t\t\t\t sizeof(event_nums),\n-\t\t\t\t\t\t event_nums,\n-\t\t\t\t\t\t (uint64_t)(uintptr_t)cq_obj);\n-\t\tif (ret) {\n-\t\t\tDRV_LOG(ERR, \"Fail to subscribe CQ to event channel.\");\n-\t\t\trte_errno = errno;\n-\t\t\tgoto error;\n-\t\t}\n-\t}\n-\t/* Initialise CQ to 1's to mark HW ownership for all CQEs. */\n-\tmemset((void *)(uintptr_t)rxq_data->cqes, 0xFF, cq_size);\n-\treturn cq_obj;\n-error:\n-\tif (cq_obj)\n-\t\tmlx5_devx_cmd_destroy(cq_obj);\n-\trxq_release_devx_cq_resources(rxq_ctrl);\n-\treturn NULL;\n-}\n-\n-/**\n- * Create the Rx hairpin queue object.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param idx\n- * Queue index in DPDK Rx queue array\n- *\n- * @return\n- * The hairpin DevX object initialised, NULL otherwise and rte_errno is set.\n- */\n-static struct mlx5_rxq_obj *\n-mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx)\n-{\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n-\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n-\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n-\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n-\tstruct mlx5_devx_create_rq_attr attr = { 0 };\n-\tstruct mlx5_rxq_obj *tmpl = NULL;\n-\tuint32_t max_wq_data;\n-\n-\tMLX5_ASSERT(rxq_data);\n-\tMLX5_ASSERT(!rxq_ctrl->obj);\n-\ttmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,\n-\t\t\t rxq_ctrl->socket);\n-\tif (!tmpl) {\n-\t\tDRV_LOG(ERR, \"port %u Rx queue %u cannot allocate resources\",\n-\t\t\tdev->data->port_id, rxq_data->idx);\n-\t\trte_errno = ENOMEM;\n-\t\treturn NULL;\n-\t}\n-\ttmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN;\n-\ttmpl->rxq_ctrl = rxq_ctrl;\n-\tattr.hairpin = 1;\n-\tmax_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz;\n-\t/* Jumbo frames > 9KB should be supported, and more packets. */\n-\tif (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) {\n-\t\tif (priv->config.log_hp_size > max_wq_data) {\n-\t\t\tDRV_LOG(ERR, \"total data size %u power of 2 is \"\n-\t\t\t\t\"too large for hairpin\",\n-\t\t\t\tpriv->config.log_hp_size);\n-\t\t\tmlx5_free(tmpl);\n-\t\t\trte_errno = ERANGE;\n-\t\t\treturn NULL;\n-\t\t}\n-\t\tattr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size;\n-\t} else {\n-\t\tattr.wq_attr.log_hairpin_data_sz =\n-\t\t\t\t(max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ?\n-\t\t\t\t max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE;\n-\t}\n-\t/* Set the packets number to the maximum value for performance. */\n-\tattr.wq_attr.log_hairpin_num_packets =\n-\t\t\tattr.wq_attr.log_hairpin_data_sz -\n-\t\t\tMLX5_HAIRPIN_QUEUE_STRIDE;\n-\ttmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &attr,\n-\t\t\t\t\t rxq_ctrl->socket);\n-\tif (!tmpl->rq) {\n-\t\tDRV_LOG(ERR,\n-\t\t\t\"port %u Rx hairpin queue %u can't create rq object\",\n-\t\t\tdev->data->port_id, idx);\n-\t\tmlx5_free(tmpl);\n-\t\trte_errno = errno;\n-\t\treturn NULL;\n-\t}\n-\tDRV_LOG(DEBUG, \"port %u rxq %u updated with %p\", dev->data->port_id,\n-\t\tidx, (void *)&tmpl);\n-\tLIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);\n-\tdev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN;\n-\treturn tmpl;\n-}\n-\n-/**\n- * Create the Rx queue Verbs/DevX object.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param idx\n- * Queue index in DPDK Rx queue array\n- * @param type\n- * Type of Rx queue object to create.\n- *\n- * @return\n- * The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.\n- */\n-struct mlx5_rxq_obj *\n-mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,\n-\t\t enum mlx5_rxq_obj_type type)\n-{\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n-\tstruct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];\n-\tstruct mlx5_rxq_ctrl *rxq_ctrl =\n-\t\tcontainer_of(rxq_data, struct mlx5_rxq_ctrl, rxq);\n-\tstruct ibv_wq_attr mod;\n-\tunsigned int cqe_n;\n-\tunsigned int wqe_n = 1 << rxq_data->elts_n;\n-\tstruct mlx5_rxq_obj *tmpl = NULL;\n-\tstruct mlx5_devx_dbr_page *cq_dbr_page = NULL;\n-\tstruct mlx5_devx_dbr_page *rq_dbr_page = NULL;\n-\tstruct mlx5dv_cq cq_info;\n-\tstruct mlx5dv_rwq rwq;\n-\tint ret = 0;\n-\tstruct mlx5dv_obj obj;\n-\n-\tMLX5_ASSERT(rxq_data);\n-\tMLX5_ASSERT(!rxq_ctrl->obj);\n-\tif (type == MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN)\n-\t\treturn mlx5_rxq_obj_hairpin_new(dev, idx);\n-\ttmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,\n-\t\t\t rxq_ctrl->socket);\n-\tif (!tmpl) {\n-\t\tDRV_LOG(ERR, \"port %u Rx queue %u cannot allocate resources\",\n-\t\t\tdev->data->port_id, rxq_data->idx);\n-\t\trte_errno = ENOMEM;\n-\t\tgoto error;\n-\t}\n-\ttmpl->type = type;\n-\ttmpl->rxq_ctrl = rxq_ctrl;\n-\tif (rxq_ctrl->irq) {\n-\t\tif (tmpl->type == MLX5_RXQ_OBJ_TYPE_IBV) {\n-\t\t\ttmpl->ibv_channel =\n-\t\t\t\tmlx5_glue->create_comp_channel(priv->sh->ctx);\n-\t\t\tif (!tmpl->ibv_channel) {\n-\t\t\t\tDRV_LOG(ERR, \"port %u: comp channel creation \"\n-\t\t\t\t\t\"failure\", dev->data->port_id);\n-\t\t\t\trte_errno = ENOMEM;\n-\t\t\t\tgoto error;\n-\t\t\t}\n-\t\t\ttmpl->fd = ((struct ibv_comp_channel *)\n-\t\t\t\t\t(tmpl->ibv_channel))->fd;\n-\t\t} else if (tmpl->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {\n-\t\t\tint devx_ev_flag =\n-\t\t\t MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA;\n-\n-\t\t\ttmpl->devx_channel =\n-\t\t\t\tmlx5_glue->devx_create_event_channel\n-\t\t\t\t\t\t\t\t(priv->sh->ctx,\n-\t\t\t\t\t\t\t\t devx_ev_flag);\n-\t\t\tif (!tmpl->devx_channel) {\n-\t\t\t\trte_errno = errno;\n-\t\t\t\tDRV_LOG(ERR,\n-\t\t\t\t\t\"Failed to create event channel %d.\",\n-\t\t\t\t\trte_errno);\n-\t\t\t\tgoto error;\n-\t\t\t}\n-\t\t\ttmpl->fd =\n-\t\t\t\tmlx5_os_get_devx_channel_fd(tmpl->devx_channel);\n-\t\t}\n-\t}\n-\tif (mlx5_rxq_mprq_enabled(rxq_data))\n-\t\tcqe_n = wqe_n * (1 << rxq_data->strd_num_n) - 1;\n-\telse\n-\t\tcqe_n = wqe_n - 1;\n-\tDRV_LOG(DEBUG, \"port %u device_attr.max_qp_wr is %d\",\n-\t\tdev->data->port_id, priv->sh->device_attr.max_qp_wr);\n-\tDRV_LOG(DEBUG, \"port %u device_attr.max_sge is %d\",\n-\t\tdev->data->port_id, priv->sh->device_attr.max_sge);\n-\tif (tmpl->type == MLX5_RXQ_OBJ_TYPE_IBV) {\n-\t\tpriv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_RX_QUEUE;\n-\t\tpriv->verbs_alloc_ctx.obj = rxq_ctrl;\n-\t\t/* Create CQ using Verbs API. */\n-\t\ttmpl->ibv_cq = mlx5_ibv_cq_new(dev, priv, rxq_data, cqe_n,\n-\t\t\t\t\t tmpl);\n-\t\tif (!tmpl->ibv_cq) {\n-\t\t\tDRV_LOG(ERR, \"port %u Rx queue %u CQ creation failure\",\n-\t\t\t\tdev->data->port_id, idx);\n-\t\t\trte_errno = ENOMEM;\n-\t\t\tgoto error;\n-\t\t}\n-\t\tobj.cq.in = tmpl->ibv_cq;\n-\t\tobj.cq.out = &cq_info;\n-\t\tret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ);\n-\t\tif (ret) {\n-\t\t\trte_errno = ret;\n-\t\t\tgoto error;\n-\t\t}\n-\t\tif (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {\n-\t\t\tDRV_LOG(ERR,\n-\t\t\t\t\"port %u wrong MLX5_CQE_SIZE environment \"\n-\t\t\t\t\"variable value: it should be set to %u\",\n-\t\t\t\tdev->data->port_id, RTE_CACHE_LINE_SIZE);\n-\t\t\trte_errno = EINVAL;\n-\t\t\tgoto error;\n-\t\t}\n-\t\t/* Fill the rings. */\n-\t\trxq_data->cqe_n = log2above(cq_info.cqe_cnt);\n-\t\trxq_data->cq_db = cq_info.dbrec;\n-\t\trxq_data->cqes =\n-\t\t\t(volatile struct mlx5_cqe (*)[])(uintptr_t)cq_info.buf;\n-\t\trxq_data->cq_uar = cq_info.cq_uar;\n-\t\trxq_data->cqn = cq_info.cqn;\n-\t\t/* Create WQ (RQ) using Verbs API. */\n-\t\ttmpl->wq = mlx5_ibv_wq_new(dev, priv, rxq_data, idx, wqe_n,\n-\t\t\t\t\t tmpl);\n-\t\tif (!tmpl->wq) {\n-\t\t\tDRV_LOG(ERR, \"port %u Rx queue %u WQ creation failure\",\n-\t\t\t\tdev->data->port_id, idx);\n-\t\t\trte_errno = ENOMEM;\n-\t\t\tgoto error;\n-\t\t}\n-\t\t/* Change queue state to ready. */\n-\t\tmod = (struct ibv_wq_attr){\n-\t\t\t.attr_mask = IBV_WQ_ATTR_STATE,\n-\t\t\t.wq_state = IBV_WQS_RDY,\n-\t\t};\n-\t\tret = mlx5_glue->modify_wq(tmpl->wq, &mod);\n-\t\tif (ret) {\n-\t\t\tDRV_LOG(ERR,\n-\t\t\t\t\"port %u Rx queue %u WQ state to IBV_WQS_RDY\"\n-\t\t\t\t\" failed\", dev->data->port_id, idx);\n-\t\t\trte_errno = ret;\n-\t\t\tgoto error;\n-\t\t}\n-\t\tobj.rwq.in = tmpl->wq;\n-\t\tobj.rwq.out = &rwq;\n-\t\tret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_RWQ);\n-\t\tif (ret) {\n-\t\t\trte_errno = ret;\n-\t\t\tgoto error;\n-\t\t}\n-\t\trxq_data->wqes = rwq.buf;\n-\t\trxq_data->rq_db = rwq.dbrec;\n-\t\tpriv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;\n-\t} else if (tmpl->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {\n-\t\tstruct mlx5_devx_modify_rq_attr rq_attr = { 0 };\n-\t\tint64_t dbr_offset;\n-\n-\t\t/* Allocate CQ door-bell. */\n-\t\tdbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs,\n-\t\t\t\t\t &cq_dbr_page);\n-\t\tif (dbr_offset < 0) {\n-\t\t\tDRV_LOG(ERR, \"Failed to allocate CQ door-bell.\");\n-\t\t\tgoto error;\n-\t\t}\n-\t\trxq_ctrl->cq_dbr_offset = dbr_offset;\n-\t\trxq_ctrl->cq_dbr_umem_id =\n-\t\t\t\t\tmlx5_os_get_umem_id(cq_dbr_page->umem);\n-\t\trxq_data->cq_db =\n-\t\t\t(uint32_t *)((uintptr_t)cq_dbr_page->dbrs +\n-\t\t\t\t (uintptr_t)rxq_ctrl->cq_dbr_offset);\n-\t\trxq_data->cq_uar =\n-\t\t\tmlx5_os_get_devx_uar_base_addr(priv->sh->devx_rx_uar);\n-\t\t/* Create CQ using DevX API. */\n-\t\ttmpl->devx_cq = mlx5_devx_cq_new(dev, cqe_n, idx, tmpl);\n-\t\tif (!tmpl->devx_cq) {\n-\t\t\tDRV_LOG(ERR, \"Failed to create CQ.\");\n-\t\t\tgoto error;\n-\t\t}\n-\t\t/* Allocate RQ door-bell. */\n-\t\tdbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs,\n-\t\t\t\t\t &rq_dbr_page);\n-\t\tif (dbr_offset < 0) {\n-\t\t\tDRV_LOG(ERR, \"Failed to allocate RQ door-bell.\");\n-\t\t\tgoto error;\n-\t\t}\n-\t\trxq_ctrl->rq_dbr_offset = dbr_offset;\n-\t\trxq_ctrl->rq_dbr_umem_id =\n-\t\t\t\t\tmlx5_os_get_umem_id(rq_dbr_page->umem);\n-\t\trxq_data->rq_db =\n-\t\t\t(uint32_t *)((uintptr_t)rq_dbr_page->dbrs +\n-\t\t\t\t (uintptr_t)rxq_ctrl->rq_dbr_offset);\n-\t\t/* Create RQ using DevX API. */\n-\t\ttmpl->rq = mlx5_devx_rq_new(dev, idx, tmpl->devx_cq->id);\n-\t\tif (!tmpl->rq) {\n-\t\t\tDRV_LOG(ERR, \"port %u Rx queue %u RQ creation failure\",\n-\t\t\t\tdev->data->port_id, idx);\n-\t\t\trte_errno = ENOMEM;\n-\t\t\tgoto error;\n-\t\t}\n-\t\t/* Change queue state to ready. */\n-\t\trq_attr.rq_state = MLX5_RQC_STATE_RST;\n-\t\trq_attr.state = MLX5_RQC_STATE_RDY;\n-\t\tret = mlx5_devx_cmd_modify_rq(tmpl->rq, &rq_attr);\n-\t\tif (ret)\n-\t\t\tgoto error;\n-\t}\n-\trxq_data->cq_arm_sn = 0;\n-\tmlx5_rxq_initialize(rxq_data);\n-\trxq_data->cq_ci = 0;\n-\tDRV_LOG(DEBUG, \"port %u rxq %u updated with %p\", dev->data->port_id,\n-\t\tidx, (void *)&tmpl);\n-\tLIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);\n-\tdev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;\n-\treturn tmpl;\n-error:\n-\tif (tmpl) {\n-\t\tret = rte_errno; /* Save rte_errno before cleanup. */\n-\t\tif (tmpl->type == MLX5_RXQ_OBJ_TYPE_IBV) {\n-\t\t\tif (tmpl->wq)\n-\t\t\t\tclaim_zero(mlx5_glue->destroy_wq(tmpl->wq));\n-\t\t\tif (tmpl->ibv_cq)\n-\t\t\t\tclaim_zero(mlx5_glue->destroy_cq(tmpl->ibv_cq));\n-\t\t\tif (tmpl->ibv_channel)\n-\t\t\t\tclaim_zero(mlx5_glue->destroy_comp_channel\n-\t\t\t\t\t\t\t(tmpl->ibv_channel));\n-\t\t\tpriv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;\n-\t\t} else if (tmpl->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {\n-\t\t\tif (tmpl->rq)\n-\t\t\t\tclaim_zero(mlx5_devx_cmd_destroy(tmpl->rq));\n-\t\t\tif (tmpl->devx_cq)\n-\t\t\t\tclaim_zero(mlx5_devx_cmd_destroy\n-\t\t\t\t\t\t\t(tmpl->devx_cq));\n-\t\t\tif (tmpl->devx_channel)\n-\t\t\t\tmlx5_glue->devx_destroy_event_channel\n-\t\t\t\t\t\t\t(tmpl->devx_channel);\n-\t\t\tif (rq_dbr_page)\n-\t\t\t\tclaim_zero(mlx5_release_dbr\n-\t\t\t\t\t\t (&priv->dbrpgs,\n-\t\t\t\t\t\t rxq_ctrl->rq_dbr_umem_id,\n-\t\t\t\t\t\t rxq_ctrl->rq_dbr_offset));\n-\t\t\tif (cq_dbr_page)\n-\t\t\t\tclaim_zero(mlx5_release_dbr\n-\t\t\t\t\t\t (&priv->dbrpgs,\n-\t\t\t\t\t\t rxq_ctrl->cq_dbr_umem_id,\n-\t\t\t\t\t\t rxq_ctrl->cq_dbr_offset));\n-\t\t}\n-\t\tmlx5_free(tmpl);\n-\t\trte_errno = ret; /* Restore rte_errno. */\n-\t}\n-\tif (type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {\n-\t\trxq_release_devx_rq_resources(rxq_ctrl);\n-\t\trxq_release_devx_cq_resources(rxq_ctrl);\n-\t}\n-\treturn NULL;\n-}\n-\n-/**\n * Verify the Rx queue objects list is empty\n *\n * @param dev\n@@ -2531,7 +1674,7 @@ struct mlx5_rxq_ctrl *\n \tif (!rte_atomic32_dec_and_test(&rxq_ctrl->refcnt))\n \t\treturn 1;\n \tif (rxq_ctrl->obj) {\n-\t\tmlx5_rxq_obj_release(rxq_ctrl->obj);\n+\t\tpriv->obj_ops->rxq_obj_release(rxq_ctrl->obj);\n \t\trxq_ctrl->obj = NULL;\n \t}\n \tif (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD)\ndiff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h\nindex b092e43..4baf5b9 100644\n--- a/drivers/net/mlx5/mlx5_rxtx.h\n+++ b/drivers/net/mlx5/mlx5_rxtx.h\n@@ -155,40 +155,12 @@ struct mlx5_rxq_data {\n \tint32_t flow_meta_offset;\n } __rte_cache_aligned;\n \n-enum mlx5_rxq_obj_type {\n-\tMLX5_RXQ_OBJ_TYPE_IBV,\t\t/* mlx5_rxq_obj with ibv_wq. */\n-\tMLX5_RXQ_OBJ_TYPE_DEVX_RQ,\t/* mlx5_rxq_obj with mlx5_devx_rq. */\n-\tMLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN,\n-\t/* mlx5_rxq_obj with mlx5_devx_rq and hairpin support. */\n-};\n-\n enum mlx5_rxq_type {\n \tMLX5_RXQ_TYPE_STANDARD, /* Standard Rx queue. */\n \tMLX5_RXQ_TYPE_HAIRPIN, /* Hairpin Rx queue. */\n \tMLX5_RXQ_TYPE_UNDEFINED,\n };\n \n-/* Verbs/DevX Rx queue elements. */\n-struct mlx5_rxq_obj {\n-\tLIST_ENTRY(mlx5_rxq_obj) next; /* Pointer to the next element. */\n-\tstruct mlx5_rxq_ctrl *rxq_ctrl; /* Back pointer to parent. */\n-\tenum mlx5_rxq_obj_type type;\n-\tint fd; /* File descriptor for event channel */\n-\tRTE_STD_C11\n-\tunion {\n-\t\tstruct {\n-\t\t\tvoid *wq; /* Work Queue. */\n-\t\t\tvoid *ibv_cq; /* Completion Queue. */\n-\t\t\tvoid *ibv_channel;\n-\t\t};\n-\t\tstruct {\n-\t\t\tstruct mlx5_devx_obj *rq; /* DevX Rx Queue object. */\n-\t\t\tstruct mlx5_devx_obj *devx_cq; /* DevX CQ object. */\n-\t\t\tvoid *devx_channel;\n-\t\t};\n-\t};\n-};\n-\n /* RX queue control descriptor. */\n struct mlx5_rxq_ctrl {\n \tstruct mlx5_rxq_data rxq; /* Data path structure. */\n@@ -416,8 +388,6 @@ int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev);\n int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);\n int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);\n-struct mlx5_rxq_obj *mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,\n-\t\t\t\t enum mlx5_rxq_obj_type type);\n int mlx5_rxq_obj_verify(struct rte_eth_dev *dev);\n struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,\n \t\t\t\t uint16_t desc, unsigned int socket,\n@@ -430,6 +400,7 @@ struct mlx5_rxq_ctrl *mlx5_rxq_hairpin_new\n int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);\n int mlx5_rxq_verify(struct rte_eth_dev *dev);\n int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl);\n+void rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl);\n int mlx5_ind_table_obj_verify(struct rte_eth_dev *dev);\n uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev,\n \t\t const uint8_t *rss_key, uint32_t rss_key_len,\ndiff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c\nindex 549af35..6376719 100644\n--- a/drivers/net/mlx5/mlx5_trigger.c\n+++ b/drivers/net/mlx5/mlx5_trigger.c\n@@ -109,16 +109,7 @@\n \tstruct mlx5_priv *priv = dev->data->dev_private;\n \tunsigned int i;\n \tint ret = 0;\n-\tenum mlx5_rxq_obj_type obj_type = MLX5_RXQ_OBJ_TYPE_IBV;\n-\tstruct mlx5_rxq_data *rxq = NULL;\n-\n-\tfor (i = 0; i < priv->rxqs_n; ++i) {\n-\t\trxq = (*priv->rxqs)[i];\n-\t\tif (rxq && rxq->lro) {\n-\t\t\tobj_type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ;\n-\t\t\tbreak;\n-\t\t}\n-\t}\n+\n \t/* Allocate/reuse/resize mempool for Multi-Packet RQ. */\n \tif (mlx5_mprq_alloc_mp(dev)) {\n \t\t/* Should not release Rx queues but return immediately. */\n@@ -130,33 +121,21 @@\n \n \t\tif (!rxq_ctrl)\n \t\t\tcontinue;\n-\t\tif (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) {\n-\t\t\trxq_ctrl->obj = mlx5_rxq_obj_new\n-\t\t\t\t(dev, i, MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN);\n-\t\t\tif (!rxq_ctrl->obj)\n+\t\tif (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD) {\n+\t\t\t/* Pre-register Rx mempool. */\n+\t\t\tmp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ?\n+\t\t\t rxq_ctrl->rxq.mprq_mp : rxq_ctrl->rxq.mp;\n+\t\t\tDRV_LOG(DEBUG, \"port %u Rx queue %u registering mp %s\"\n+\t\t\t\t\" having %u chunks\", dev->data->port_id,\n+\t\t\t\trxq_ctrl->rxq.idx, mp->name, mp->nb_mem_chunks);\n+\t\t\tmlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp);\n+\t\t\tret = rxq_alloc_elts(rxq_ctrl);\n+\t\t\tif (ret)\n \t\t\t\tgoto error;\n-\t\t\tcontinue;\n \t\t}\n-\t\t/* Pre-register Rx mempool. */\n-\t\tmp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ?\n-\t\t rxq_ctrl->rxq.mprq_mp : rxq_ctrl->rxq.mp;\n-\t\tDRV_LOG(DEBUG,\n-\t\t\t\"port %u Rx queue %u registering\"\n-\t\t\t\" mp %s having %u chunks\",\n-\t\t\tdev->data->port_id, rxq_ctrl->rxq.idx,\n-\t\t\tmp->name, mp->nb_mem_chunks);\n-\t\tmlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp);\n-\t\tret = rxq_alloc_elts(rxq_ctrl);\n-\t\tif (ret)\n-\t\t\tgoto error;\n-\t\trxq_ctrl->obj = mlx5_rxq_obj_new(dev, i, obj_type);\n+\t\trxq_ctrl->obj = priv->obj_ops->rxq_obj_new(dev, i);\n \t\tif (!rxq_ctrl->obj)\n \t\t\tgoto error;\n-\t\tif (obj_type == MLX5_RXQ_OBJ_TYPE_IBV)\n-\t\t\trxq_ctrl->wqn =\n-\t\t\t\t((struct ibv_wq *)(rxq_ctrl->obj->wq))->wq_num;\n-\t\telse if (obj_type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ)\n-\t\t\trxq_ctrl->wqn = rxq_ctrl->obj->rq->id;\n \t}\n \treturn 0;\n error:\n", "prefixes": [ "v1", "05/18" ] }