Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/89479/?format=api
{ "id": 89479, "url": "https://patches.dpdk.org/api/patches/89479/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/1616061368-29768-2-git-send-email-michaelba@nvidia.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1616061368-29768-2-git-send-email-michaelba@nvidia.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1616061368-29768-2-git-send-email-michaelba@nvidia.com", "date": "2021-03-18T09:56:07", "name": "[1/2] net/mlx5: workaround ASO memory region creation", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "1b9a4e949e5131df5191cfaaf15910701df6e9b1", "submitter": { "id": 1949, "url": "https://patches.dpdk.org/api/people/1949/?format=api", "name": "Michael Baum", "email": "michaelba@nvidia.com" }, "delegate": { "id": 3268, "url": "https://patches.dpdk.org/api/users/3268/?format=api", "username": "rasland", "first_name": "Raslan", "last_name": "Darawsheh", "email": "rasland@nvidia.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/1616061368-29768-2-git-send-email-michaelba@nvidia.com/mbox/", "series": [ { "id": 15764, "url": "https://patches.dpdk.org/api/series/15764/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=15764", "date": "2021-03-18T09:56:08", "name": "adjusting mkey creations", "version": 1, "mbox": "https://patches.dpdk.org/series/15764/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/89479/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/89479/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 4206BA0561;\n\tThu, 18 Mar 2021 10:56:37 +0100 (CET)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 84030140F93;\n\tThu, 18 Mar 2021 10:56:26 +0100 (CET)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n by mails.dpdk.org (Postfix) with ESMTP id D1FE7140F6F\n for <dev@dpdk.org>; Thu, 18 Mar 2021 10:56:22 +0100 (CET)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n michaelba@nvidia.com) with SMTP; 18 Mar 2021 11:56:18 +0200", "from nvidia.com (pegasus07.mtr.labs.mlnx [10.210.16.112])\n by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12I9uHRE019444;\n Thu, 18 Mar 2021 11:56:18 +0200" ], "From": "Michael Baum <michaelba@nvidia.com>", "To": "dev@dpdk.org", "Cc": "Matan Azrad <matan@nvidia.com>, Raslan Darawsheh <rasland@nvidia.com>,\n Viacheslav Ovsiienko <viacheslavo@nvidia.com>, stable@dpdk.org", "Date": "Thu, 18 Mar 2021 09:56:07 +0000", "Message-Id": "<1616061368-29768-2-git-send-email-michaelba@nvidia.com>", "X-Mailer": "git-send-email 1.8.3.1", "In-Reply-To": "<1616061368-29768-1-git-send-email-michaelba@nvidia.com>", "References": "<1616061368-29768-1-git-send-email-michaelba@nvidia.com>", "Subject": "[dpdk-dev] [PATCH 1/2] net/mlx5: workaround ASO memory region\n creation", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Due to kernel issue in direct MKEY creation using the DevX API for\nphysical memory, this patch replaces the ASO MR creation to use Verbs\nAPI.\n\nFixes: f935ed4b645a (\"net/mlx5: support flow hit action for aging\")\nCc: stable@dpdk.org\n\nSigned-off-by: Michael Baum <michaelba@nvidia.com>\nAcked-by: Matan Azrad <matan@nvidia.com>\n---\n drivers/common/mlx5/linux/mlx5_common_verbs.c | 1 -\n drivers/common/mlx5/windows/mlx5_common_os.c | 23 +++---\n drivers/net/mlx5/mlx5.h | 10 +--\n drivers/net/mlx5/mlx5_flow_age.c | 106 +++++++++++---------------\n 4 files changed, 58 insertions(+), 82 deletions(-)", "diff": "diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c\nindex 339535d..aa560f0 100644\n--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c\n+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c\n@@ -37,7 +37,6 @@\n {\n \tstruct ibv_mr *ibv_mr;\n \n-\tmemset(pmd_mr, 0, sizeof(*pmd_mr));\n \tibv_mr = mlx5_glue->reg_mr(pd, addr, length,\n \t\t\t\t IBV_ACCESS_LOCAL_WRITE |\n \t\t\t\t (haswell_broadwell_cpu ? 0 :\ndiff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c\nindex f2d781a..cebf42d 100644\n--- a/drivers/common/mlx5/windows/mlx5_common_os.c\n+++ b/drivers/common/mlx5/windows/mlx5_common_os.c\n@@ -155,23 +155,22 @@\n \tstruct mlx5_devx_mkey_attr mkey_attr;\n \tstruct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd;\n \tstruct mlx5_hca_attr attr;\n+\tstruct mlx5_devx_obj *mkey;\n+\tvoid *obj;\n \n \tif (!pd || !addr) {\n \t\trte_errno = EINVAL;\n \t\treturn -1;\n \t}\n-\tmemset(pmd_mr, 0, sizeof(*pmd_mr));\n \tif (mlx5_devx_cmd_query_hca_attr(mlx5_pd->devx_ctx, &attr))\n \t\treturn -1;\n-\tpmd_mr->addr = addr;\n-\tpmd_mr->len = length;\n-\tpmd_mr->obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, pmd_mr->addr,\n-\t\t\t\t pmd_mr->len, IBV_ACCESS_LOCAL_WRITE);\n-\tif (!pmd_mr->obj)\n+\tobj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, addr, length,\n+\t\t\t IBV_ACCESS_LOCAL_WRITE);\n+\tif (!obj)\n \t\treturn -1;\n \tmkey_attr.addr = (uintptr_t)addr;\n \tmkey_attr.size = length;\n-\tmkey_attr.umem_id = ((struct mlx5_devx_umem *)(pmd_mr->obj))->umem_id;\n+\tmkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id;\n \tmkey_attr.pd = mlx5_pd->pdn;\n \tmkey_attr.log_entity_size = 0;\n \tmkey_attr.pg_access = 0;\n@@ -183,11 +182,15 @@\n \t\tmkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write;\n \t\tmkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read;\n \t}\n-\tpmd_mr->mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr);\n-\tif (!pmd_mr->mkey) {\n-\t\tclaim_zero(mlx5_os_umem_dereg(pmd_mr->obj));\n+\tmkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr);\n+\tif (!mkey) {\n+\t\tclaim_zero(mlx5_os_umem_dereg(obj));\n \t\treturn -1;\n \t}\n+\tpmd_mr->addr = addr;\n+\tpmd_mr->len = length;\n+\tpmd_mr->obj = obj;\n+\tpmd_mr->mkey = mkey;\n \tpmd_mr->lkey = pmd_mr->mkey->id;\n \treturn 0;\n }\ndiff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h\nindex 14043b6..e2eb4db 100644\n--- a/drivers/net/mlx5/mlx5.h\n+++ b/drivers/net/mlx5/mlx5.h\n@@ -471,14 +471,6 @@ struct mlx5_aso_cq {\n \tuint64_t errors;\n };\n \n-struct mlx5_aso_devx_mr {\n-\tvoid *buf;\n-\tuint64_t length;\n-\tstruct mlx5dv_devx_umem *umem;\n-\tstruct mlx5_devx_obj *mkey;\n-\tbool is_indirect;\n-};\n-\n struct mlx5_aso_sq_elem {\n \tstruct mlx5_aso_age_pool *pool;\n \tuint16_t burst_size;\n@@ -489,7 +481,7 @@ struct mlx5_aso_sq {\n \tstruct mlx5_aso_cq cq;\n \tstruct mlx5_devx_sq sq_obj;\n \tvolatile uint64_t *uar_addr;\n-\tstruct mlx5_aso_devx_mr mr;\n+\tstruct mlx5_pmd_mr mr;\n \tuint16_t pi;\n \tuint32_t head;\n \tuint32_t tail;\ndiff --git a/drivers/net/mlx5/mlx5_flow_age.c b/drivers/net/mlx5/mlx5_flow_age.c\nindex 00cb20d..c0be7c3 100644\n--- a/drivers/net/mlx5/mlx5_flow_age.c\n+++ b/drivers/net/mlx5/mlx5_flow_age.c\n@@ -61,90 +61,72 @@\n /**\n * Free MR resources.\n *\n+ * @param[in] sh\n+ * Pointer to shared device context.\n * @param[in] mr\n * MR to free.\n */\n static void\n-mlx5_aso_devx_dereg_mr(struct mlx5_aso_devx_mr *mr)\n+mlx5_aso_dereg_mr(struct mlx5_dev_ctx_shared *sh, struct mlx5_pmd_mr *mr)\n {\n-\tclaim_zero(mlx5_devx_cmd_destroy(mr->mkey));\n-\tif (!mr->is_indirect && mr->umem)\n-\t\tclaim_zero(mlx5_glue->devx_umem_dereg(mr->umem));\n-\tmlx5_free(mr->buf);\n+\tvoid *addr = mr->addr;\n+\n+\tsh->share_cache.dereg_mr_cb(mr);\n+\tmlx5_free(addr);\n \tmemset(mr, 0, sizeof(*mr));\n }\n \n /**\n * Register Memory Region.\n *\n- * @param[in] ctx\n- * Context returned from mlx5 open_device() glue function.\n+ * @param[in] sh\n+ * Pointer to shared device context.\n * @param[in] length\n * Size of MR buffer.\n * @param[in/out] mr\n * Pointer to MR to create.\n * @param[in] socket\n * Socket to use for allocation.\n- * @param[in] pdn\n- * Protection Domain number to use.\n *\n * @return\n * 0 on success, a negative errno value otherwise and rte_errno is set.\n */\n static int\n-mlx5_aso_devx_reg_mr(void *ctx, size_t length, struct mlx5_aso_devx_mr *mr,\n-\t\t int socket, int pdn)\n+mlx5_aso_reg_mr(struct mlx5_dev_ctx_shared *sh, size_t length,\n+\t\tstruct mlx5_pmd_mr *mr, int socket)\n {\n-\tstruct mlx5_devx_mkey_attr mkey_attr;\n \n-\tmr->buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096,\n-\t\t\t socket);\n-\tif (!mr->buf) {\n-\t\tDRV_LOG(ERR, \"Failed to create ASO bits mem for MR by Devx.\");\n+\tint ret;\n+\n+\tmr->addr = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096,\n+\t\t\t socket);\n+\tif (!mr->addr) {\n+\t\tDRV_LOG(ERR, \"Failed to create ASO bits mem for MR.\");\n \t\treturn -1;\n \t}\n-\tmr->umem = mlx5_os_umem_reg(ctx, mr->buf, length,\n-\t\t\t\t\t\t IBV_ACCESS_LOCAL_WRITE);\n-\tif (!mr->umem) {\n-\t\tDRV_LOG(ERR, \"Failed to register Umem for MR by Devx.\");\n-\t\tgoto error;\n-\t}\n-\tmkey_attr.addr = (uintptr_t)mr->buf;\n-\tmkey_attr.size = length;\n-\tmkey_attr.umem_id = mlx5_os_get_umem_id(mr->umem);\n-\tmkey_attr.pd = pdn;\n-\tmkey_attr.pg_access = 1;\n-\tmkey_attr.klm_array = NULL;\n-\tmkey_attr.klm_num = 0;\n-\tmkey_attr.relaxed_ordering_read = 0;\n-\tmkey_attr.relaxed_ordering_write = 0;\n-\tmr->mkey = mlx5_devx_cmd_mkey_create(ctx, &mkey_attr);\n-\tif (!mr->mkey) {\n+\tret = sh->share_cache.reg_mr_cb(sh->pd, mr->addr, length, mr);\n+\tif (ret) {\n \t\tDRV_LOG(ERR, \"Failed to create direct Mkey.\");\n-\t\tgoto error;\n+\t\tmlx5_free(mr->addr);\n+\t\treturn -1;\n \t}\n-\tmr->length = length;\n-\tmr->is_indirect = false;\n \treturn 0;\n-error:\n-\tif (mr->umem)\n-\t\tclaim_zero(mlx5_glue->devx_umem_dereg(mr->umem));\n-\tmlx5_free(mr->buf);\n-\treturn -1;\n }\n \n /**\n * Destroy Send Queue used for ASO access.\n *\n+ * @param[in] sh\n+ * Pointer to shared device context.\n * @param[in] sq\n * ASO SQ to destroy.\n */\n static void\n-mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq)\n+mlx5_aso_destroy_sq(struct mlx5_dev_ctx_shared *sh, struct mlx5_aso_sq *sq)\n {\n \tmlx5_devx_sq_destroy(&sq->sq_obj);\n \tmlx5_aso_cq_destroy(&sq->cq);\n-\tmlx5_aso_devx_dereg_mr(&sq->mr);\n+\tmlx5_aso_dereg_mr(sh, &sq->mr);\n \tmemset(sq, 0, sizeof(*sq));\n }\n \n@@ -166,8 +148,8 @@\n \tfor (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {\n \t\twqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |\n \t\t\t\t\t\t\t (sizeof(*wqe) >> 4));\n-\t\twqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.mkey->id);\n-\t\taddr = (uint64_t)((uint64_t *)sq->mr.buf + i *\n+\t\twqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);\n+\t\taddr = (uint64_t)((uint64_t *)sq->mr.addr + i *\n \t\t\t\t\t MLX5_ASO_AGE_ACTIONS_PER_POOL / 64);\n \t\twqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));\n \t\twqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);\n@@ -184,31 +166,31 @@\n /**\n * Create Send Queue used for ASO access.\n *\n- * @param[in] ctx\n- * Context returned from mlx5 open_device() glue function.\n+ * @param[in] sh\n+ * Pointer to shared device context.\n * @param[in/out] sq\n * Pointer to SQ to create.\n * @param[in] socket\n * Socket to use for allocation.\n * @param[in] uar\n * User Access Region object.\n- * @param[in] pdn\n- * Protection Domain number to use.\n * @param[in] log_desc_n\n * Log of number of descriptors in queue.\n+ * @param[in] ts_format\n+ * timestamp format supported by the queue.\n *\n * @return\n * 0 on success, a negative errno value otherwise and rte_errno is set.\n */\n static int\n-mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket,\n-\t\t void *uar, uint32_t pdn, uint16_t log_desc_n,\n+mlx5_aso_sq_create(struct mlx5_dev_ctx_shared *sh, struct mlx5_aso_sq *sq,\n+\t\t int socket, void *uar, uint16_t log_desc_n,\n \t\t uint32_t ts_format)\n {\n \tstruct mlx5_devx_create_sq_attr attr = {\n \t\t.user_index = 0xFFFF,\n \t\t.wq_attr = (struct mlx5_devx_wq_attr){\n-\t\t\t.pd = pdn,\n+\t\t\t.pd = sh->pdn,\n \t\t\t.uar_page = mlx5_os_get_devx_uar_page_id(uar),\n \t\t},\n \t\t.ts_format = mlx5_ts_format_conv(ts_format),\n@@ -220,17 +202,18 @@\n \tuint16_t log_wqbb_n;\n \tint ret;\n \n-\tif (mlx5_aso_devx_reg_mr(ctx, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *\n-\t\t\t\t sq_desc_n, &sq->mr, socket, pdn))\n+\tif (mlx5_aso_reg_mr(sh, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * sq_desc_n,\n+\t\t\t &sq->mr, socket))\n \t\treturn -1;\n-\tif (mlx5_aso_cq_create(ctx, &sq->cq, log_desc_n, socket,\n+\tif (mlx5_aso_cq_create(sh->ctx, &sq->cq, log_desc_n, socket,\n \t\t\t mlx5_os_get_devx_uar_page_id(uar)))\n \t\tgoto error;\n \tsq->log_desc_n = log_desc_n;\n \tattr.cqn = sq->cq.cq_obj.cq->id;\n \t/* for mlx5_aso_wqe that is twice the size of mlx5_wqe */\n \tlog_wqbb_n = log_desc_n + 1;\n-\tret = mlx5_devx_sq_create(ctx, &sq->sq_obj, log_wqbb_n, &attr, socket);\n+\tret = mlx5_devx_sq_create(sh->ctx, &sq->sq_obj, log_wqbb_n, &attr,\n+\t\t\t\t socket);\n \tif (ret) {\n \t\tDRV_LOG(ERR, \"Can't create SQ object.\");\n \t\trte_errno = ENOMEM;\n@@ -250,7 +233,7 @@\n \tmlx5_aso_init_sq(sq);\n \treturn 0;\n error:\n-\tmlx5_aso_destroy_sq(sq);\n+\tmlx5_aso_destroy_sq(sh, sq);\n \treturn -1;\n }\n \n@@ -266,9 +249,8 @@\n int\n mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh)\n {\n-\treturn mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0,\n-\t\t\t\t sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC,\n-\t\t\t\t sh->sq_ts_format);\n+\treturn mlx5_aso_sq_create(sh, &sh->aso_age_mng->aso_sq, 0, sh->tx_uar,\n+\t\t\t\t MLX5_ASO_QUEUE_LOG_DESC, sh->sq_ts_format);\n }\n \n /**\n@@ -280,7 +262,7 @@\n void\n mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh)\n {\n-\tmlx5_aso_destroy_sq(&sh->aso_age_mng->aso_sq);\n+\tmlx5_aso_destroy_sq(sh, &sh->aso_age_mng->aso_sq);\n }\n \n /**\n@@ -410,7 +392,7 @@\n \t\tuint16_t idx = (sq->tail + i) & mask;\n \t\tstruct mlx5_aso_age_pool *pool = sq->elts[idx].pool;\n \t\tuint64_t diff = curr - pool->time_of_last_age_check;\n-\t\tuint64_t *addr = sq->mr.buf;\n+\t\tuint64_t *addr = sq->mr.addr;\n \t\tint j;\n \n \t\taddr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64;\n", "prefixes": [ "1/2" ] }