Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/74621/?format=api
{ "id": 74621, "url": "https://patches.dpdk.org/api/patches/74621/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/1595429948-20873-1-git-send-email-viacheslavo@mellanox.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1595429948-20873-1-git-send-email-viacheslavo@mellanox.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1595429948-20873-1-git-send-email-viacheslavo@mellanox.com", "date": "2020-07-22T14:59:08", "name": "net/mlx5: fix UAR memory mapping type", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "46d7c646db088f4b056938f04434cae7b95adaae", "submitter": { "id": 1102, "url": "https://patches.dpdk.org/api/people/1102/?format=api", "name": "Slava Ovsiienko", "email": "viacheslavo@mellanox.com" }, "delegate": { "id": 3268, "url": "https://patches.dpdk.org/api/users/3268/?format=api", "username": "rasland", "first_name": "Raslan", "last_name": "Darawsheh", "email": "rasland@nvidia.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/1595429948-20873-1-git-send-email-viacheslavo@mellanox.com/mbox/", "series": [ { "id": 11238, "url": "https://patches.dpdk.org/api/series/11238/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=11238", "date": "2020-07-22T14:59:08", "name": "net/mlx5: fix UAR memory mapping type", "version": 1, "mbox": "https://patches.dpdk.org/series/11238/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/74621/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/74621/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 1339BA0526;\n\tWed, 22 Jul 2020 16:59:19 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id D2B1E1BFCA;\n\tWed, 22 Jul 2020 16:59:17 +0200 (CEST)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n by dpdk.org (Postfix) with ESMTP id 77C0A1BFBB\n for <dev@dpdk.org>; Wed, 22 Jul 2020 16:59:15 +0200 (CEST)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n viacheslavo@mellanox.com) with SMTP; 22 Jul 2020 17:59:10 +0300", "from pegasus12.mtr.labs.mlnx (pegasus12.mtr.labs.mlnx\n [10.210.17.40])\n by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 06MExABH013767;\n Wed, 22 Jul 2020 17:59:10 +0300", "from pegasus12.mtr.labs.mlnx (localhost [127.0.0.1])\n by pegasus12.mtr.labs.mlnx (8.14.7/8.14.7) with ESMTP id 06MExAPR020911;\n Wed, 22 Jul 2020 14:59:10 GMT", "(from viacheslavo@localhost)\n by pegasus12.mtr.labs.mlnx (8.14.7/8.14.7/Submit) id 06MEx9lr020910;\n Wed, 22 Jul 2020 14:59:09 GMT" ], "X-Authentication-Warning": "pegasus12.mtr.labs.mlnx: viacheslavo set sender to\n viacheslavo@mellanox.com using -f", "From": "Viacheslav Ovsiienko <viacheslavo@mellanox.com>", "To": "dev@dpdk.org", "Cc": "matan@mellanox.com, rasland@mellanox.com, thomas@monjalon.net,\n dekelp@mellanox.com", "Date": "Wed, 22 Jul 2020 14:59:08 +0000", "Message-Id": "<1595429948-20873-1-git-send-email-viacheslavo@mellanox.com>", "X-Mailer": "git-send-email 1.8.3.1", "Subject": "[dpdk-dev] [PATCH] net/mlx5: fix UAR memory mapping type", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "The User Access Region is a special mechanism to provide direct\naccess to the hardware registers, and is the part of PCI address\nspace that is mapped to CPU virtual address. The mapping can be\nperformed with the type \"Write-Combining\" or \"Non-Cached\", and\nthese ones might be supported or not on different setups.\n\nTo prevent device probing failure the UAR allocation attempt\nwith alternative mapping type is performed. The datapath\ntakes the actual UAR mapping into account on queue creation.\n\nThere was another issue with NULL UAR base address.\nOFED 5.0.x and Upstream rdma_core before v29 returned the NULL as\nUAR base address if UAR was not the first object in the UAR page.\nIt caused the PMD failure and we should try to get another UAR\ntill we get the first one with non-NULL base address returned.\n\nFixes: fc4d4f732bbc (\"net/mlx5: introduce shared UAR resource\")\n\nSigned-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>\n---\n drivers/net/mlx5/mlx5.c | 160 ++++++++++++++++++++++++++++++++++++++-----\n drivers/net/mlx5/mlx5_defs.h | 11 +++\n 2 files changed, 153 insertions(+), 18 deletions(-)", "diff": "diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c\nindex 6c7a7ee..a5cccd1 100644\n--- a/drivers/net/mlx5/mlx5.c\n+++ b/drivers/net/mlx5/mlx5.c\n@@ -706,6 +706,141 @@ struct mlx5_flow_id_pool *\n \tprf->obj = NULL;\n }\n \n+/*\n+ * Allocate Rx and Tx UARs in robust fashion.\n+ * This routine handles the following UAR allocation issues:\n+ *\n+ * - tries to allocate the UAR with the most appropriate memory\n+ * mapping type from the ones supported by the host\n+ *\n+ * - tries to allocate the UAR with non-NULL base address\n+ * OFED 5.0.x and Upstream rdma_core before v29 returned the NULL as\n+ * UAR base address if UAR was not the first object in the UAR page.\n+ * It caused the PMD failure and we should try to get another UAR\n+ * till we get the first one with non-NULL base address returned.\n+ */\n+static int\n+mlx5_alloc_rxtx_uars(struct mlx5_dev_ctx_shared *sh,\n+\t\t const struct mlx5_dev_config *config)\n+{\n+\tuint32_t uar_mapping, retry;\n+\tint err = 0;\n+\n+\tfor (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) {\n+#ifdef MLX5DV_UAR_ALLOC_TYPE_NC\n+\t\t/* Control the mapping type according to the settings. */\n+\t\tuar_mapping = (config->dbnc == MLX5_TXDB_NCACHED) ?\n+\t\t\t MLX5DV_UAR_ALLOC_TYPE_NC :\n+\t\t\t MLX5DV_UAR_ALLOC_TYPE_BF;\n+#else\n+\t\tRTE_SET_USED(config);\n+\t\t/*\n+\t\t * It seems we have no way to control the memory mapping type\n+\t\t * for the UAR, the default \"Write-Combining\" type is supposed.\n+\t\t * The UAR initialization on queue creation queries the\n+\t\t * actual mapping type done by Verbs/kernel and setups the\n+\t\t * PMD datapath accordingly.\n+\t\t */\n+\t\tuar_mapping = 0;\n+#endif\n+\t\tsh->tx_uar = mlx5_glue->devx_alloc_uar(sh->ctx, uar_mapping);\n+#ifdef MLX5DV_UAR_ALLOC_TYPE_NC\n+\t\tif (!sh->tx_uar &&\n+\t\t uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) {\n+\t\t\tif (config->dbnc == MLX5_TXDB_CACHED ||\n+\t\t\t config->dbnc == MLX5_TXDB_HEURISTIC)\n+\t\t\t\tDRV_LOG(WARNING, \"Devarg tx_db_nc setting \"\n+\t\t\t\t\t\t \"is not supported by DevX\");\n+\t\t\t/*\n+\t\t\t * In some environments like virtual machine\n+\t\t\t * the Write Combining mapped might be not supported\n+\t\t\t * and UAR allocation fails. We try \"Non-Cached\"\n+\t\t\t * mapping for the case. The tx_burst routines take\n+\t\t\t * the UAR mapping type into account on UAR setup\n+\t\t\t * on queue creation.\n+\t\t\t */\n+\t\t\tDRV_LOG(WARNING, \"Failed to allocate Tx DevX UAR (BF)\");\n+\t\t\tuar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC;\n+\t\t\tsh->tx_uar = mlx5_glue->devx_alloc_uar\n+\t\t\t\t\t\t\t(sh->ctx, uar_mapping);\n+\t\t} else if (!sh->tx_uar &&\n+\t\t\t uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) {\n+\t\t\tif (config->dbnc == MLX5_TXDB_NCACHED)\n+\t\t\t\tDRV_LOG(WARNING, \"Devarg tx_db_nc settings \"\n+\t\t\t\t\t\t \"is not supported by DevX\");\n+\t\t\t/*\n+\t\t\t * If Verbs/kernel does not support \"Non-Cached\"\n+\t\t\t * try the \"Write-Combining\".\n+\t\t\t */\n+\t\t\tDRV_LOG(WARNING, \"Failed to allocate Tx DevX UAR (NC)\");\n+\t\t\tuar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF;\n+\t\t\tsh->tx_uar = mlx5_glue->devx_alloc_uar\n+\t\t\t\t\t\t\t(sh->ctx, uar_mapping);\n+\t\t}\n+#endif\n+\t\tif (!sh->tx_uar) {\n+\t\t\tDRV_LOG(ERR, \"Failed to allocate Tx DevX UAR (BF/NC)\");\n+\t\t\terr = ENOMEM;\n+\t\t\tgoto exit;\n+\t\t}\n+\t\tif (sh->tx_uar->base_addr)\n+\t\t\tbreak;\n+\t\t/*\n+\t\t * The UARs are allocated by rdma_core within the\n+\t\t * IB device context, on context closure all UARs\n+\t\t * will be freed, should be no memory/object leakage.\n+\t\t */\n+\t\tDRV_LOG(WARNING, \"Retrying to allocate Tx DevX UAR\");\n+\t\tsh->tx_uar = NULL;\n+\t}\n+\t/* Check whether we finally succeeded with valid UAR allocation. */\n+\tif (!sh->tx_uar) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate Tx DevX UAR (NULL base)\");\n+\t\terr = ENOMEM;\n+\t\tgoto exit;\n+\t}\n+\tfor (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) {\n+\t\tuar_mapping = 0;\n+\t\tsh->devx_rx_uar = mlx5_glue->devx_alloc_uar\n+\t\t\t\t\t\t\t(sh->ctx, uar_mapping);\n+#ifdef MLX5DV_UAR_ALLOC_TYPE_NC\n+\t\tif (!sh->devx_rx_uar &&\n+\t\t uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) {\n+\t\t\t/*\n+\t\t\t * Rx UAR is used to control interrupts only,\n+\t\t\t * should be no datapath noticeable impact,\n+\t\t\t * can try \"Non-Cached\" mapping safely.\n+\t\t\t */\n+\t\t\tDRV_LOG(WARNING, \"Failed to allocate Rx DevX UAR (BF)\");\n+\t\t\tuar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC;\n+\t\t\tsh->devx_rx_uar = mlx5_glue->devx_alloc_uar\n+\t\t\t\t\t\t\t(sh->ctx, uar_mapping);\n+\t\t}\n+#endif\n+\t\tif (!sh->devx_rx_uar) {\n+\t\t\tDRV_LOG(ERR, \"Failed to allocate Rx DevX UAR (BF/NC)\");\n+\t\t\terr = ENOMEM;\n+\t\t\tgoto exit;\n+\t\t}\n+\t\tif (sh->devx_rx_uar->base_addr)\n+\t\t\tbreak;\n+\t\t/*\n+\t\t * The UARs are allocated by rdma_core within the\n+\t\t * IB device context, on context closure all UARs\n+\t\t * will be freed, should be no memory/object leakage.\n+\t\t */\n+\t\tDRV_LOG(WARNING, \"Retrying to allocate Rx DevX UAR\");\n+\t\tsh->devx_rx_uar = NULL;\n+\t}\n+\t/* Check whether we finally succeeded with valid UAR allocation. */\n+\tif (!sh->devx_rx_uar) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate Rx DevX UAR (NULL base)\");\n+\t\terr = ENOMEM;\n+\t}\n+exit:\n+\treturn err;\n+}\n+\n /**\n * Allocate shared device context. If there is multiport device the\n * master and representors will share this context, if there is single\n@@ -807,18 +942,11 @@ struct mlx5_dev_ctx_shared *\n \t\t\terr = ENOMEM;\n \t\t\tgoto error;\n \t\t}\n-\t\tsh->tx_uar = mlx5_glue->devx_alloc_uar(sh->ctx, 0);\n-\t\tif (!sh->tx_uar) {\n-\t\t\tDRV_LOG(ERR, \"Failed to allocate DevX UAR.\");\n-\t\t\terr = ENOMEM;\n-\t\t\tgoto error;\n-\t\t}\n-\t\tsh->devx_rx_uar = mlx5_glue->devx_alloc_uar(sh->ctx, 0);\n-\t\tif (!sh->devx_rx_uar) {\n-\t\t\tDRV_LOG(ERR, \"Failed to allocate Rx DevX UAR.\");\n-\t\t\terr = ENOMEM;\n+\t\terr = mlx5_alloc_rxtx_uars(sh, config);\n+\t\tif (err)\n \t\t\tgoto error;\n-\t\t}\n+\t\tMLX5_ASSERT(sh->tx_uar && sh->tx_uar->base_addr);\n+\t\tMLX5_ASSERT(sh->devx_rx_uar && sh->devx_rx_uar->base_addr);\n \t}\n \tsh->flow_id_pool = mlx5_flow_id_pool_alloc\n \t\t\t\t\t((1 << HAIRPIN_FLOW_ID_BITS) - 1);\n@@ -874,20 +1002,16 @@ struct mlx5_dev_ctx_shared *\n \tpthread_mutex_destroy(&sh->txpp.mutex);\n \tpthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);\n \tMLX5_ASSERT(sh);\n-\tif (sh->cnt_id_tbl) {\n+\tif (sh->cnt_id_tbl)\n \t\tmlx5_l3t_destroy(sh->cnt_id_tbl);\n-\t\tsh->cnt_id_tbl = NULL;\n-\t}\n-\tif (sh->tx_uar) {\n-\t\tmlx5_glue->devx_free_uar(sh->tx_uar);\n-\t\tsh->tx_uar = NULL;\n-\t}\n \tif (sh->tis)\n \t\tclaim_zero(mlx5_devx_cmd_destroy(sh->tis));\n \tif (sh->td)\n \t\tclaim_zero(mlx5_devx_cmd_destroy(sh->td));\n \tif (sh->devx_rx_uar)\n \t\tmlx5_glue->devx_free_uar(sh->devx_rx_uar);\n+\tif (sh->tx_uar)\n+\t\tmlx5_glue->devx_free_uar(sh->tx_uar);\n \tif (sh->pd)\n \t\tclaim_zero(mlx5_glue->dealloc_pd(sh->pd));\n \tif (sh->ctx)\ndiff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h\nindex 7ed3e88..e5f7acc 100644\n--- a/drivers/net/mlx5/mlx5_defs.h\n+++ b/drivers/net/mlx5/mlx5_defs.h\n@@ -196,4 +196,15 @@\n #define static_assert _Static_assert\n #endif\n \n+/*\n+ * Defines the amount of retries to allocate the first UAR in the page.\n+ * OFED 5.0.x and Upstream rdma_core before v29 returned the NULL as\n+ * UAR base address if UAR was not the first object in the UAR page.\n+ * It caused the PMD failure and we should try to get another UAR\n+ * till we get the first one with non-NULL base address returned.\n+ * Should follow the rdma_core internal (not exported) definition\n+ * MLX5_NUM_NON_FP_BFREGS_PER_UAR.\n+ */\n+#define MLX5_ALLOC_UAR_RETRY 2\n+\n #endif /* RTE_PMD_MLX5_DEFS_H_ */\n", "prefixes": [] }