Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/65300/?format=api
{ "id": 65300, "url": "http://patches.dpdk.org/api/patches/65300/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1580292549-27439-14-git-send-email-matan@mellanox.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1580292549-27439-14-git-send-email-matan@mellanox.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1580292549-27439-14-git-send-email-matan@mellanox.com", "date": "2020-01-29T10:09:09", "name": "[v2,13/13] vdpa/mlx5: disable ROCE", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "fec113fb5a1148eabd376cd803df39c898ef7e73", "submitter": { "id": 796, "url": "http://patches.dpdk.org/api/people/796/?format=api", "name": "Matan Azrad", "email": "matan@mellanox.com" }, "delegate": { "id": 2642, "url": "http://patches.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1580292549-27439-14-git-send-email-matan@mellanox.com/mbox/", "series": [ { "id": 8335, "url": "http://patches.dpdk.org/api/series/8335/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=8335", "date": "2020-01-29T10:08:56", "name": "Introduce mlx5 vDPA driver", "version": 2, "mbox": "http://patches.dpdk.org/series/8335/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/65300/comments/", "check": "fail", "checks": "http://patches.dpdk.org/api/patches/65300/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id CC9EEA0531;\n\tWed, 29 Jan 2020 11:11:50 +0100 (CET)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 519791BFD6;\n\tWed, 29 Jan 2020 11:10:31 +0100 (CET)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n by dpdk.org (Postfix) with ESMTP id 8144F1C029\n for <dev@dpdk.org>; Wed, 29 Jan 2020 11:10:29 +0100 (CET)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n asafp@mellanox.com)\n with ESMTPS (AES256-SHA encrypted); 29 Jan 2020 12:10:27 +0200", "from pegasus07.mtr.labs.mlnx (pegasus07.mtr.labs.mlnx\n [10.210.16.112])\n by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 00TA9BHT032108;\n Wed, 29 Jan 2020 12:10:27 +0200" ], "From": "Matan Azrad <matan@mellanox.com>", "To": "dev@dpdk.org, Viacheslav Ovsiienko <viacheslavo@mellanox.com>", "Cc": "Maxime Coquelin <maxime.coquelin@redhat.com>", "Date": "Wed, 29 Jan 2020 10:09:09 +0000", "Message-Id": "<1580292549-27439-14-git-send-email-matan@mellanox.com>", "X-Mailer": "git-send-email 1.8.3.1", "In-Reply-To": "<1580292549-27439-1-git-send-email-matan@mellanox.com>", "References": "<1579539790-3882-1-git-send-email-matan@mellanox.com>\n <1580292549-27439-1-git-send-email-matan@mellanox.com>", "Subject": "[dpdk-dev] [PATCH v2 13/13] vdpa/mlx5: disable ROCE", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "In order to support virtio queue creation by the FW, ROCE mode\nshould be disabled in the device.\n\nDo it by netlink which is like the devlink tool commands:\n\t1. devlink dev param set pci/[pci] name enable_roce value false\n\t cmode driverinit\n \t2. devlink dev reload pci/[pci]\nOr by sysfs which is like:\n\techo 0 > /sys/bus/pci/devices/[pci]/roce_enable\n\nThe IB device is matched again after ROCE disabling.\n\nSigned-off-by: Matan Azrad <matan@mellanox.com>\nAcked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>\n---\n drivers/vdpa/mlx5/Makefile | 2 +-\n drivers/vdpa/mlx5/meson.build | 2 +-\n drivers/vdpa/mlx5/mlx5_vdpa.c | 192 ++++++++++++++++++++++++++++++++++--------\n 3 files changed, 161 insertions(+), 35 deletions(-)", "diff": "diff --git a/drivers/vdpa/mlx5/Makefile b/drivers/vdpa/mlx5/Makefile\nindex 4d1f528..5cdcdd4 100644\n--- a/drivers/vdpa/mlx5/Makefile\n+++ b/drivers/vdpa/mlx5/Makefile\n@@ -29,7 +29,7 @@ CFLAGS += -D_XOPEN_SOURCE=600\n CFLAGS += $(WERROR_FLAGS)\n CFLAGS += -Wno-strict-prototypes\n LDLIBS += -lrte_common_mlx5\n-LDLIBS += -lrte_eal -lrte_vhost -lrte_kvargs -lrte_bus_pci -lrte_sched\n+LDLIBS += -lrte_eal -lrte_vhost -lrte_kvargs -lrte_pci -lrte_bus_pci -lrte_sched\n \n # A few warnings cannot be avoided in external headers.\n CFLAGS += -Wno-error=cast-qual\ndiff --git a/drivers/vdpa/mlx5/meson.build b/drivers/vdpa/mlx5/meson.build\nindex 2e521b8..e5236b3 100644\n--- a/drivers/vdpa/mlx5/meson.build\n+++ b/drivers/vdpa/mlx5/meson.build\n@@ -9,7 +9,7 @@ endif\n \n fmt_name = 'mlx5_vdpa'\n allow_experimental_apis = true\n-deps += ['hash', 'common_mlx5', 'vhost', 'bus_pci', 'eal', 'sched']\n+deps += ['hash', 'common_mlx5', 'vhost', 'pci', 'bus_pci', 'eal', 'sched']\n sources = files(\n \t'mlx5_vdpa.c',\n \t'mlx5_vdpa_mem.c',\ndiff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c\nindex c8ef3b4..e098be9 100644\n--- a/drivers/vdpa/mlx5/mlx5_vdpa.c\n+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c\n@@ -1,15 +1,19 @@\n /* SPDX-License-Identifier: BSD-3-Clause\n * Copyright 2019 Mellanox Technologies, Ltd\n */\n+#include <unistd.h>\n+\n #include <rte_malloc.h>\n #include <rte_log.h>\n #include <rte_errno.h>\n #include <rte_bus_pci.h>\n+#include <rte_pci.h>\n \n #include <mlx5_glue.h>\n #include <mlx5_common.h>\n #include <mlx5_devx_cmds.h>\n #include <mlx5_prm.h>\n+#include <mlx5_nl.h>\n \n #include \"mlx5_vdpa_utils.h\"\n #include \"mlx5_vdpa.h\"\n@@ -228,6 +232,145 @@\n \t.get_notify_area = NULL,\n };\n \n+static struct ibv_device *\n+mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)\n+{\n+\tint n;\n+\tstruct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);\n+\tstruct ibv_device *ibv_match = NULL;\n+\n+\tif (!ibv_list) {\n+\t\trte_errno = ENOSYS;\n+\t\treturn NULL;\n+\t}\n+\twhile (n-- > 0) {\n+\t\tstruct rte_pci_addr pci_addr;\n+\n+\t\tDRV_LOG(DEBUG, \"Checking device \\\"%s\\\"..\", ibv_list[n]->name);\n+\t\tif (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))\n+\t\t\tcontinue;\n+\t\tif (memcmp(addr, &pci_addr, sizeof(pci_addr)))\n+\t\t\tcontinue;\n+\t\tibv_match = ibv_list[n];\n+\t\tbreak;\n+\t}\n+\tif (!ibv_match)\n+\t\trte_errno = ENOENT;\n+\tmlx5_glue->free_device_list(ibv_list);\n+\treturn ibv_match;\n+}\n+\n+/* Try to disable ROCE by Netlink\\Devlink. */\n+static int\n+mlx5_vdpa_nl_roce_disable(const char *addr)\n+{\n+\tint nlsk_fd = mlx5_nl_init(NETLINK_GENERIC);\n+\tint devlink_id;\n+\tint enable;\n+\tint ret;\n+\n+\tif (nlsk_fd < 0)\n+\t\treturn nlsk_fd;\n+\tdevlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);\n+\tif (devlink_id < 0) {\n+\t\tret = devlink_id;\n+\t\tDRV_LOG(DEBUG, \"Failed to get devlink id for ROCE operations by\"\n+\t\t\t\" Netlink.\");\n+\t\tgoto close;\n+\t}\n+\tret = mlx5_nl_enable_roce_get(nlsk_fd, devlink_id, addr, &enable);\n+\tif (ret) {\n+\t\tDRV_LOG(DEBUG, \"Failed to get ROCE enable by Netlink: %d.\",\n+\t\t\tret);\n+\t\tgoto close;\n+\t} else if (!enable) {\n+\t\tDRV_LOG(INFO, \"ROCE has already disabled(Netlink).\");\n+\t\tgoto close;\n+\t}\n+\tret = mlx5_nl_enable_roce_set(nlsk_fd, devlink_id, addr, 0);\n+\tif (ret)\n+\t\tDRV_LOG(DEBUG, \"Failed to disable ROCE by Netlink: %d.\", ret);\n+\telse\n+\t\tDRV_LOG(INFO, \"ROCE is disabled by Netlink successfully.\");\n+close:\n+\tclose(nlsk_fd);\n+\treturn ret;\n+}\n+\n+/* Try to disable ROCE by sysfs. */\n+static int\n+mlx5_vdpa_sys_roce_disable(const char *addr)\n+{\n+\tFILE *file_o;\n+\tint enable;\n+\tint ret;\n+\n+\tMKSTR(file_p, \"/sys/bus/pci/devices/%s/roce_enable\", addr);\n+\tfile_o = fopen(file_p, \"rb\");\n+\tif (!file_o) {\n+\t\trte_errno = ENOTSUP;\n+\t\treturn -ENOTSUP;\n+\t}\n+\tret = fscanf(file_o, \"%d\", &enable);\n+\tif (ret != 1) {\n+\t\trte_errno = EINVAL;\n+\t\tret = EINVAL;\n+\t\tgoto close;\n+\t} else if (!enable) {\n+\t\tret = 0;\n+\t\tDRV_LOG(INFO, \"ROCE has already disabled(sysfs).\");\n+\t\tgoto close;\n+\t}\n+\tfclose(file_o);\n+\tfile_o = fopen(file_p, \"wb\");\n+\tif (!file_o) {\n+\t\trte_errno = ENOTSUP;\n+\t\treturn -ENOTSUP;\n+\t}\n+\tfprintf(file_o, \"0\\n\");\n+\tret = 0;\n+close:\n+\tif (ret)\n+\t\tDRV_LOG(DEBUG, \"Failed to disable ROCE by sysfs: %d.\", ret);\n+\telse\n+\t\tDRV_LOG(INFO, \"ROCE is disabled by sysfs successfully.\");\n+\tfclose(file_o);\n+\treturn ret;\n+}\n+\n+#define MLX5_VDPA_MAX_RETRIES 20\n+#define MLX5_VDPA_USEC 1000\n+static int\n+mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)\n+{\n+\tchar addr_name[64] = {0};\n+\n+\trte_pci_device_name(addr, addr_name, sizeof(addr_name));\n+\t/* Firstly try to disable ROCE by Netlink and fallback to sysfs. */\n+\tif (mlx5_vdpa_nl_roce_disable(addr_name) == 0 ||\n+\t mlx5_vdpa_sys_roce_disable(addr_name) == 0) {\n+\t\t/*\n+\t\t * Succeed to disable ROCE, wait for the IB device to appear\n+\t\t * again after reload.\n+\t\t */\n+\t\tint r;\n+\t\tstruct ibv_device *ibv_new;\n+\n+\t\tfor (r = MLX5_VDPA_MAX_RETRIES; r; r--) {\n+\t\t\tibv_new = mlx5_vdpa_get_ib_device_match(addr);\n+\t\t\tif (ibv_new) {\n+\t\t\t\t*ibv = ibv_new;\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\tusleep(MLX5_VDPA_USEC);\n+\t\t}\n+\t\tDRV_LOG(ERR, \"Cannot much device %s after ROCE disable, \"\n+\t\t\t\"retries exceed %d\", addr_name, MLX5_VDPA_MAX_RETRIES);\n+\t\trte_errno = EAGAIN;\n+\t}\n+\treturn -rte_errno;\n+}\n+\n /**\n * DPDK callback to register a PCI device.\n *\n@@ -245,8 +388,7 @@\n mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,\n \t\t struct rte_pci_device *pci_dev __rte_unused)\n {\n-\tstruct ibv_device **ibv_list;\n-\tstruct ibv_device *ibv_match = NULL;\n+\tstruct ibv_device *ibv;\n \tstruct mlx5_vdpa_priv *priv = NULL;\n \tstruct ibv_context *ctx = NULL;\n \tstruct mlx5_hca_attr attr;\n@@ -257,42 +399,26 @@\n \t\t\t\" driver.\");\n \t\treturn 1;\n \t}\n-\terrno = 0;\n-\tibv_list = mlx5_glue->get_device_list(&ret);\n-\tif (!ibv_list) {\n-\t\trte_errno = errno;\n-\t\tDRV_LOG(ERR, \"Failed to get device list, is ib_uverbs loaded?\");\n-\t\treturn -ENOSYS;\n-\t}\n-\twhile (ret-- > 0) {\n-\t\tstruct rte_pci_addr pci_addr;\n-\n-\t\tDRV_LOG(DEBUG, \"Checking device \\\"%s\\\"..\", ibv_list[ret]->name);\n-\t\tif (mlx5_dev_to_pci_addr(ibv_list[ret]->ibdev_path, &pci_addr))\n-\t\t\tcontinue;\n-\t\tif (pci_dev->addr.domain != pci_addr.domain ||\n-\t\t pci_dev->addr.bus != pci_addr.bus ||\n-\t\t pci_dev->addr.devid != pci_addr.devid ||\n-\t\t pci_dev->addr.function != pci_addr.function)\n-\t\t\tcontinue;\n-\t\tDRV_LOG(INFO, \"PCI information matches for device \\\"%s\\\".\",\n-\t\t\tibv_list[ret]->name);\n-\t\tibv_match = ibv_list[ret];\n-\t\tbreak;\n-\t}\n-\tmlx5_glue->free_device_list(ibv_list);\n-\tif (!ibv_match) {\n+\tibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);\n+\tif (!ibv) {\n \t\tDRV_LOG(ERR, \"No matching IB device for PCI slot \"\n-\t\t\t\"%\" SCNx32 \":%\" SCNx8 \":%\" SCNx8 \".%\" SCNx8 \".\",\n-\t\t\tpci_dev->addr.domain, pci_dev->addr.bus,\n-\t\t\tpci_dev->addr.devid, pci_dev->addr.function);\n+\t\t\tPCI_PRI_FMT \".\", pci_dev->addr.domain,\n+\t\t\tpci_dev->addr.bus, pci_dev->addr.devid,\n+\t\t\tpci_dev->addr.function);\n \t\trte_errno = ENOENT;\n \t\treturn -rte_errno;\n+\t} else {\n+\t\tDRV_LOG(INFO, \"PCI information matches for device \\\"%s\\\".\",\n+\t\t\tibv->name);\n+\t}\n+\tif (mlx5_vdpa_roce_disable(&pci_dev->addr, &ibv) != 0) {\n+\t\tDRV_LOG(WARNING, \"Failed to disable ROCE for \\\"%s\\\".\",\n+\t\t\tibv->name);\n+\t\t//return -rte_errno;\n \t}\n-\tctx = mlx5_glue->dv_open_device(ibv_match);\n+\tctx = mlx5_glue->dv_open_device(ibv);\n \tif (!ctx) {\n-\t\tDRV_LOG(ERR, \"Failed to open IB device \\\"%s\\\".\",\n-\t\t\tibv_match->name);\n+\t\tDRV_LOG(ERR, \"Failed to open IB device \\\"%s\\\".\", ibv->name);\n \t\trte_errno = ENODEV;\n \t\treturn -rte_errno;\n \t}\n", "prefixes": [ "v2", "13/13" ] }