Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/64953/?format=api
{ "id": 64953, "url": "http://patches.dpdk.org/api/patches/64953/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1579539790-3882-21-git-send-email-matan@mellanox.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1579539790-3882-21-git-send-email-matan@mellanox.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1579539790-3882-21-git-send-email-matan@mellanox.com", "date": "2020-01-20T17:02:52", "name": "[v1,20/38] vdpa/mlx5: prepare completion queues", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "467fed54e36876b22f0baf31caf5d24905a7d3a2", "submitter": { "id": 796, "url": "http://patches.dpdk.org/api/people/796/?format=api", "name": "Matan Azrad", "email": "matan@mellanox.com" }, "delegate": { "id": 2642, "url": "http://patches.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1579539790-3882-21-git-send-email-matan@mellanox.com/mbox/", "series": [ { "id": 8223, "url": "http://patches.dpdk.org/api/series/8223/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=8223", "date": "2020-01-20T17:02:37", "name": "Introduce mlx5 vDPA driver", "version": 1, "mbox": "http://patches.dpdk.org/series/8223/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/64953/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/64953/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 2CCD9A0526;\n\tMon, 20 Jan 2020 18:06:01 +0100 (CET)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 510F51BFFE;\n\tMon, 20 Jan 2020 18:03:46 +0100 (CET)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n by dpdk.org (Postfix) with ESMTP id 4D3211BFA4\n for <dev@dpdk.org>; Mon, 20 Jan 2020 18:03:14 +0100 (CET)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n asafp@mellanox.com)\n with ESMTPS (AES256-SHA encrypted); 20 Jan 2020 19:03:13 +0200", "from pegasus07.mtr.labs.mlnx (pegasus07.mtr.labs.mlnx\n [10.210.16.112])\n by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 00KH3BGd024424;\n Mon, 20 Jan 2020 19:03:13 +0200" ], "From": "Matan Azrad <matan@mellanox.com>", "To": "dev@dpdk.org", "Cc": "Maxime Coquelin <maxime.coquelin@redhat.com>,\n Thomas Monjalon <thomas@monjalon.net>", "Date": "Mon, 20 Jan 2020 17:02:52 +0000", "Message-Id": "<1579539790-3882-21-git-send-email-matan@mellanox.com>", "X-Mailer": "git-send-email 1.8.3.1", "In-Reply-To": "<1579539790-3882-1-git-send-email-matan@mellanox.com>", "References": "<1579539790-3882-1-git-send-email-matan@mellanox.com>", "Subject": "[dpdk-dev] [PATCH v1 20/38] vdpa/mlx5: prepare completion queues", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "As an arrangement to the vitrio queues creation, a CQ should be created\nfor the virtio queue.\n\nThe design is to trigger an event for the guest and for the vdpa driver\nwhen a new CQE is posted by the HW after the packet transition.\n\nThis patch add the basic operations to create and destroy CQs and to\ntrigger the CQE events when a new CQE is posted.\n\nSigned-off-by: Matan Azrad <matan@mellanox.com>\n---\n drivers/vdpa/mlx5/Makefile | 1 +\n drivers/vdpa/mlx5/meson.build | 1 +\n drivers/vdpa/mlx5/mlx5_vdpa.h | 56 ++++++++++++++\n drivers/vdpa/mlx5/mlx5_vdpa_cq.c | 154 +++++++++++++++++++++++++++++++++++++++\n 4 files changed, 212 insertions(+)\n create mode 100644 drivers/vdpa/mlx5/mlx5_vdpa_cq.c", "diff": "diff --git a/drivers/vdpa/mlx5/Makefile b/drivers/vdpa/mlx5/Makefile\nindex 5472797..f813824 100644\n--- a/drivers/vdpa/mlx5/Makefile\n+++ b/drivers/vdpa/mlx5/Makefile\n@@ -9,6 +9,7 @@ LIB = librte_pmd_mlx5_vdpa.a\n # Sources.\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_VDPA_PMD) += mlx5_vdpa.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_VDPA_PMD) += mlx5_vdpa_mem.c\n+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_VDPA_PMD) += mlx5_vdpa_cq.c\n \n # Basic CFLAGS.\n CFLAGS += -O3\ndiff --git a/drivers/vdpa/mlx5/meson.build b/drivers/vdpa/mlx5/meson.build\nindex 7e5dd95..aec5d34 100644\n--- a/drivers/vdpa/mlx5/meson.build\n+++ b/drivers/vdpa/mlx5/meson.build\n@@ -13,6 +13,7 @@ deps += ['hash', 'common_mlx5', 'vhost', 'bus_pci', 'eal', 'sched']\n sources = files(\n \t'mlx5_vdpa.c',\n \t'mlx5_vdpa_mem.c',\n+\t'mlx5_vdpa_cq.c',\n )\n cflags_options = [\n \t'-std=c11',\ndiff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h\nindex e27baea..6008e3f 100644\n--- a/drivers/vdpa/mlx5/mlx5_vdpa.h\n+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h\n@@ -9,9 +9,27 @@\n \n #include <rte_vdpa.h>\n #include <rte_vhost.h>\n+#include <rte_spinlock.h>\n \n #include <mlx5_glue.h>\n #include <mlx5_devx_cmds.h>\n+#include <mlx5_prm.h>\n+\n+\n+struct mlx5_vdpa_cq {\n+\tuint16_t log_desc_n;\n+\tuint32_t cq_ci:24;\n+\tuint32_t arm_sn:2;\n+\trte_spinlock_t sl;\n+\tstruct mlx5_devx_obj *cq;\n+\tstruct mlx5dv_devx_umem *umem_obj;\n+\tunion {\n+\t\tvolatile void *umem_buf;\n+\t\tvolatile struct mlx5_cqe *cqes;\n+\t};\n+\tvolatile uint32_t *db_rec;\n+\tuint64_t errors;\n+};\n \n struct mlx5_vdpa_query_mr {\n \tSLIST_ENTRY(mlx5_vdpa_query_mr) next;\n@@ -34,6 +52,9 @@ struct mlx5_vdpa_priv {\n \tuint32_t gpa_mkey_index;\n \tstruct ibv_mr *null_mr;\n \tstruct rte_vhost_memory *vmem;\n+\tuint32_t eqn;\n+\tstruct mlx5dv_devx_event_channel *eventc;\n+\tstruct mlx5dv_devx_uar *uar;\n \tSLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list;\n };\n \n@@ -57,4 +78,39 @@ struct mlx5_vdpa_priv {\n */\n int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv);\n \n+\n+/**\n+ * Create a CQ and all its related resources.\n+ *\n+ * @param[in] priv\n+ * The vdpa driver private structure.\n+ * @param[in] desc_n\n+ * Number of CQEs.\n+ * @param[in] callfd\n+ * The guest notification file descriptor.\n+ * @param[in/out] cq\n+ * Pointer to the CQ structure.\n+ *\n+ * @return\n+ * 0 on success, -1 otherwise and rte_errno is set.\n+ */\n+int mlx5_vdpa_cq_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,\n+\t\t\tint callfd, struct mlx5_vdpa_cq *cq);\n+\n+/**\n+ * Destroy a CQ and all its related resources.\n+ *\n+ * @param[in/out] cq\n+ * Pointer to the CQ structure.\n+ */\n+void mlx5_vdpa_cq_destroy(struct mlx5_vdpa_cq *cq);\n+\n+/**\n+ * Release all the CQ global resources.\n+ *\n+ * @param[in] priv\n+ * The vdpa driver private structure.\n+ */\n+void mlx5_vdpa_cq_global_release(struct mlx5_vdpa_priv *priv);\n+\n #endif /* RTE_PMD_MLX5_VDPA_H_ */\ndiff --git a/drivers/vdpa/mlx5/mlx5_vdpa_cq.c b/drivers/vdpa/mlx5/mlx5_vdpa_cq.c\nnew file mode 100644\nindex 0000000..563277f\n--- /dev/null\n+++ b/drivers/vdpa/mlx5/mlx5_vdpa_cq.c\n@@ -0,0 +1,154 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2019 Mellanox Technologies, Ltd\n+ */\n+#include <unistd.h>\n+#include <stdint.h>\n+#include <assert.h>\n+\n+#include <rte_malloc.h>\n+#include <rte_errno.h>\n+#include <rte_lcore.h>\n+\n+#include \"mlx5_vdpa_utils.h\"\n+#include \"mlx5_vdpa.h\"\n+\n+void\n+mlx5_vdpa_cq_global_release(struct mlx5_vdpa_priv *priv)\n+{\n+\tif (priv->uar) {\n+\t\tmlx5_glue->devx_free_uar(priv->uar);\n+\t\tpriv->uar = NULL;\n+\t}\n+\tif (priv->eventc) {\n+\t\tmlx5_glue->devx_destroy_event_channel(priv->eventc);\n+\t\tpriv->eventc = NULL;\n+\t}\n+\tpriv->eqn = 0;\n+}\n+\n+/* Prepare all the global resources for all the CQs.*/\n+static int\n+mlx5_vdpa_cq_global_prepare(struct mlx5_vdpa_priv *priv)\n+{\n+\tuint32_t lcore;\n+\n+\tif (priv->eventc)\n+\t\treturn 0;\n+\tlcore = (uint32_t)rte_lcore_to_cpu_id(-1);\n+\tif (mlx5_glue->devx_query_eqn(priv->ctx, lcore, &priv->eqn)) {\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to query EQ number %d.\", rte_errno);\n+\t\treturn -1;\n+\t}\n+\tpriv->eventc = mlx5_glue->devx_create_event_channel(priv->ctx,\n+\t\t\t MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA);\n+\tif (!priv->eventc) {\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to create event channel %d.\",\n+\t\t\trte_errno);\n+\t\tgoto error;\n+\t}\n+\tpriv->uar = mlx5_glue->devx_alloc_uar(priv->ctx, 0);\n+\tif (!priv->uar) {\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to allocate UAR.\");\n+\t\tgoto error;\n+\t}\n+\treturn 0;\n+error:\n+\tmlx5_vdpa_cq_global_release(priv);\n+\treturn -1;\n+}\n+\n+void\n+mlx5_vdpa_cq_destroy(struct mlx5_vdpa_cq *cq)\n+{\n+\tint ret __rte_unused;\n+\n+\tif (cq->cq) {\n+\t\tret = mlx5_devx_cmd_destroy(cq->cq);\n+\t\tassert(!ret);\n+\t}\n+\tif (cq->umem_obj) {\n+\t\tret = mlx5_glue->devx_umem_dereg(cq->umem_obj);\n+\t\tassert(!ret);\n+\t}\n+\tif (cq->umem_buf)\n+\t\trte_free((void *)(uintptr_t)cq->umem_buf);\n+\tmemset(cq, 0, sizeof(*cq));\n+}\n+\n+int\n+mlx5_vdpa_cq_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n, int callfd,\n+\t\t struct mlx5_vdpa_cq *cq)\n+{\n+\tstruct mlx5_devx_cq_attr attr;\n+\tsize_t pgsize = sysconf(_SC_PAGESIZE);\n+\tuint32_t log_desc_n = rte_log2_u32(desc_n);\n+\tuint32_t umem_size;\n+\tint ret;\n+\tuint16_t event_nums[1] = {0};\n+\n+\tif (mlx5_vdpa_cq_global_prepare(priv))\n+\t\treturn -1;\n+\tcq->log_desc_n = log_desc_n;\n+\tumem_size = sizeof(struct mlx5_cqe) * (1 << log_desc_n) +\n+\t\t\t\t\t\t\tsizeof(*cq->db_rec) * 2;\n+\tcq->umem_buf = rte_zmalloc(__func__, umem_size, 4096);\n+\tif (!cq->umem_buf) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate memory for CQ.\");\n+\t\trte_errno = ENOMEM;\n+\t\treturn -ENOMEM;\n+\t}\n+\tcq->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,\n+\t\t\t\t\t\t(void *)(uintptr_t)cq->umem_buf,\n+\t\t\t\t\t\tumem_size,\n+\t\t\t\t\t\tIBV_ACCESS_LOCAL_WRITE);\n+\tif (!cq->umem_obj) {\n+\t\tDRV_LOG(ERR, \"Failed to register umem for CQ.\");\n+\t\tgoto error;\n+\t}\n+\tattr.q_umem_valid = 1;\n+\tattr.db_umem_valid = 1;\n+\tattr.use_first_only = 0;\n+\tattr.overrun_ignore = 0;\n+\tattr.uar_page_id = priv->uar->page_id;\n+\tattr.q_umem_id = cq->umem_obj->umem_id;\n+\tattr.q_umem_offset = 0;\n+\tattr.db_umem_id = cq->umem_obj->umem_id;\n+\tattr.db_umem_offset = sizeof(struct mlx5_cqe) * (1 << log_desc_n);\n+\tattr.eqn = priv->eqn;\n+\tattr.log_cq_size = log_desc_n;\n+\tattr.log_page_size = rte_log2_u32(pgsize);\n+\tcq->cq = mlx5_devx_cmd_create_cq(priv->ctx, &attr);\n+\tif (!cq->cq)\n+\t\tgoto error;\n+\tcq->db_rec = RTE_PTR_ADD(cq->umem_buf, (uintptr_t)attr.db_umem_offset);\n+\tcq->cq_ci = 0;\n+\trte_spinlock_init(&cq->sl);\n+\t/* Subscribe CQ event to the event channel controlled by the driver. */\n+\tret = mlx5_glue->devx_subscribe_devx_event(priv->eventc, cq->cq->obj,\n+\t\t\t\t\t\t sizeof(event_nums),\n+\t\t\t\t\t\t event_nums,\n+\t\t\t\t\t\t (uint64_t)(uintptr_t)cq);\n+\tif (ret) {\n+\t\tDRV_LOG(ERR, \"Failed to subscribe CQE event.\");\n+\t\trte_errno = errno;\n+\t\tgoto error;\n+\t}\n+\t/* Subscribe CQ event to the guest FD only if it is not in poll mode. */\n+\tif (callfd != -1) {\n+\t\tret = mlx5_glue->devx_subscribe_devx_event_fd(priv->eventc,\n+\t\t\t\t\t\t\t callfd,\n+\t\t\t\t\t\t\t cq->cq->obj, 0);\n+\t\tif (ret) {\n+\t\t\tDRV_LOG(ERR, \"Failed to subscribe CQE event fd.\");\n+\t\t\trte_errno = errno;\n+\t\t\tgoto error;\n+\t\t}\n+\t}\n+\treturn 0;\n+error:\n+\tmlx5_vdpa_cq_destroy(cq);\n+\treturn -1;\n+}\n", "prefixes": [ "v1", "20/38" ] }