Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/45864/?format=api
{ "id": 45864, "url": "http://patches.dpdk.org/api/patches/45864/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1538484302-97153-5-git-send-email-jia.guo@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1538484302-97153-5-git-send-email-jia.guo@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1538484302-97153-5-git-send-email-jia.guo@intel.com", "date": "2018-10-02T12:45:02", "name": "[v3,4/4] vfio: enable vfio hotplug by req notifier handler", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "4a7db7701b4f303fa8425c336bcf727739bfc7d5", "submitter": { "id": 507, "url": "http://patches.dpdk.org/api/people/507/?format=api", "name": "Guo, Jia", "email": "jia.guo@intel.com" }, "delegate": null, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1538484302-97153-5-git-send-email-jia.guo@intel.com/mbox/", "series": [ { "id": 1638, "url": "http://patches.dpdk.org/api/series/1638/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=1638", "date": "2018-10-02T12:44:59", "name": "Enable hotplug in vfio", "version": 3, "mbox": "http://patches.dpdk.org/series/1638/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/45864/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/45864/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 989785B36;\n\tTue, 2 Oct 2018 14:48:27 +0200 (CEST)", "from mga03.intel.com (mga03.intel.com [134.134.136.65])\n\tby dpdk.org (Postfix) with ESMTP id B58F5200\n\tfor <dev@dpdk.org>; Tue, 2 Oct 2018 14:48:20 +0200 (CEST)", "from orsmga006.jf.intel.com ([10.7.209.51])\n\tby orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t02 Oct 2018 05:48:19 -0700", "from jeffguo-s2600wt2.sh.intel.com (HELO localhost.localdomain)\n\t([10.67.110.10])\n\tby orsmga006.jf.intel.com with ESMTP; 02 Oct 2018 05:41:33 -0700" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.54,332,1534834800\"; d=\"scan'208\";a=\"79210145\"", "From": "Jeff Guo <jia.guo@intel.com>", "To": "stephen@networkplumber.org, bruce.richardson@intel.com,\n\tferruh.yigit@intel.com, konstantin.ananyev@intel.com,\n\tgaetan.rivet@6wind.com, jingjing.wu@intel.com, thomas@monjalon.net,\n\tmotih@mellanox.com, matan@mellanox.com, harry.van.haaren@intel.com,\n\tqi.z.zhang@intel.com, shaopeng.he@intel.com,\n\tbernard.iremonger@intel.com, \n\tarybchenko@solarflare.com, anatoly.burakov@intel.com", "Cc": "jblunck@infradead.org, shreyansh.jain@nxp.com, dev@dpdk.org,\n\tjia.guo@intel.com, helin.zhang@intel.com, jerin.jacob@caviumnetworks.com", "Date": "Tue, 2 Oct 2018 20:45:02 +0800", "Message-Id": "<1538484302-97153-5-git-send-email-jia.guo@intel.com>", "X-Mailer": "git-send-email 2.7.4", "In-Reply-To": "<1538484302-97153-1-git-send-email-jia.guo@intel.com>", "References": "<1534503091-31910-1-git-send-email-jia.guo@intel.com>\n\t<1538484302-97153-1-git-send-email-jia.guo@intel.com>", "Subject": "[dpdk-dev] [PATCH v3 4/4] vfio: enable vfio hotplug by req notifier\n\thandler", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "When device is be hot-unplugged, the vfio kernel module will sent req\nnotifier to request user space to release the allocated resources at\nfirst. After that, vfio kernel module will detect the device disappear,\nand then delete the device in kernel.\n\nThis patch aim to add req notifier processing to enable hotplug for vfio.\nBy enable the req notifier monitoring and register the notifier callback,\nwhen device be hot-unplugged, the hot-unplug handler will be called to\nprocess hotplug for vfio.\n\nSigned-off-by: Jeff Guo <jia.guo@intel.com>\n---\nv3->v2:\nchange some code style and typo\n---\n drivers/bus/pci/linux/pci_vfio.c | 111 +++++++++++++++++++++++++++++++++++++++\n drivers/bus/pci/pci_common.c | 10 ++++\n 2 files changed, 121 insertions(+)", "diff": "diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c\nindex 686386d..5d3d026 100644\n--- a/drivers/bus/pci/linux/pci_vfio.c\n+++ b/drivers/bus/pci/linux/pci_vfio.c\n@@ -17,6 +17,8 @@\n #include <rte_eal_memconfig.h>\n #include <rte_malloc.h>\n #include <rte_vfio.h>\n+#include <rte_eal.h>\n+#include <rte_bus.h>\n \n #include \"eal_filesystem.h\"\n \n@@ -277,6 +279,101 @@ pci_vfio_setup_interrupts(struct rte_pci_device *dev, int vfio_dev_fd)\n \treturn -1;\n }\n \n+static void\n+pci_vfio_req_handler(void *param)\n+{\n+\tstruct rte_bus *bus;\n+\tint ret;\n+\tstruct rte_device *device = (struct rte_device *)param;\n+\n+\tbus = rte_bus_find_by_device(device);\n+\tif (bus == NULL) {\n+\t\tRTE_LOG(ERR, EAL, \"Cannot find bus for device (%s)\\n\",\n+\t\t\tdevice->name);\n+\t\treturn;\n+\t}\n+\n+\t/*\n+\t * vfio kernel module request user space to release allocated\n+\t * resources before device be deleted in kernel, so it can directly\n+\t * call the vfio bus hot-unplug handler to process it.\n+\t */\n+\tret = bus->hot_unplug_handler(device);\n+\tif (ret)\n+\t\tRTE_LOG(ERR, EAL,\n+\t\t\t\"Can not handle hot-unplug for device (%s)\\n\",\n+\t\t\tdevice->name);\n+}\n+\n+/* enable notifier (only enable req now) */\n+static int\n+pci_vfio_enable_notifier(struct rte_pci_device *dev, int vfio_dev_fd)\n+{\n+\tint ret;\n+\tint fd = -1;\n+\n+\t/* set up an eventfd for req notifier */\n+\tfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);\n+\tif (fd < 0) {\n+\t\tRTE_LOG(ERR, EAL, \"Cannot set up eventfd, error %i (%s)\\n\",\n+\t\t\terrno, strerror(errno));\n+\t\treturn -1;\n+\t}\n+\n+\tdev->req_notifier_handler.fd = fd;\n+\tdev->req_notifier_handler.type = RTE_INTR_HANDLE_VFIO_REQ;\n+\tdev->req_notifier_handler.vfio_dev_fd = vfio_dev_fd;\n+\tret = rte_intr_callback_register(&dev->req_notifier_handler,\n+\t\t\t\t\t pci_vfio_req_handler,\n+\t\t\t\t\t (void *)&dev->device);\n+\tif (ret) {\n+\t\tRTE_LOG(ERR, EAL, \"Fail to register req notifier handler.\\n\");\n+\t\tgoto error;\n+\t}\n+\n+\tret = rte_intr_enable(&dev->req_notifier_handler);\n+\tif (ret) {\n+\t\tRTE_LOG(ERR, EAL, \"Fail to enable req notifier.\\n\");\n+\t\tret = rte_intr_callback_unregister(&dev->req_notifier_handler,\n+\t\t\t\t\t\t pci_vfio_req_handler,\n+\t\t\t\t\t\t (void *)&dev->device);\n+\t\tif (ret)\n+\t\t\tRTE_LOG(ERR, EAL,\n+\t\t\t\t\"Fail to unregister req notifier handler.\\n\");\n+\t\tgoto error;\n+\t}\n+\n+\treturn 0;\n+error:\n+\tclose(fd);\n+\treturn -1;\n+}\n+\n+/* disable notifier (only disable req now) */\n+static int\n+pci_vfio_disable_notifier(struct rte_pci_device *dev)\n+{\n+\tint ret;\n+\n+\tret = rte_intr_disable(&dev->req_notifier_handler);\n+\tif (ret) {\n+\t\tRTE_LOG(ERR, EAL, \"fail to disable req notifier.\\n\");\n+\t\treturn -1;\n+\t}\n+\n+\tret = rte_intr_callback_unregister(&dev->req_notifier_handler,\n+\t\t\t\t\t pci_vfio_req_handler,\n+\t\t\t\t\t (void *)&dev->device);\n+\tif (ret) {\n+\t\tRTE_LOG(ERR, EAL,\n+\t\t\t \"fail to unregister req notifier handler.\\n\");\n+\t\treturn -1;\n+\t}\n+\n+\tclose(dev->req_notifier_handler.fd);\n+\treturn 0;\n+}\n+\n static int\n pci_vfio_is_ioport_bar(int vfio_dev_fd, int bar_index)\n {\n@@ -430,6 +527,7 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)\n \tstruct pci_map *maps;\n \n \tdev->intr_handle.fd = -1;\n+\tdev->req_notifier_handler.fd = -1;\n \n \t/* store PCI address string */\n \tsnprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,\n@@ -521,6 +619,11 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)\n \t\tgoto err_vfio_res;\n \t}\n \n+\tif (pci_vfio_enable_notifier(dev, vfio_dev_fd) != 0) {\n+\t\tRTE_LOG(ERR, EAL, \"Error setting up notifier!\\n\");\n+\t\tgoto err_vfio_res;\n+\t}\n+\n \tTAILQ_INSERT_TAIL(vfio_res_list, vfio_res, next);\n \n \treturn 0;\n@@ -546,6 +649,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)\n \tstruct pci_map *maps;\n \n \tdev->intr_handle.fd = -1;\n+\tdev->req_notifier_handler.fd = -1;\n \n \t/* store PCI address string */\n \tsnprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,\n@@ -586,6 +690,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)\n \n \t/* we need save vfio_dev_fd, so it can be used during release */\n \tdev->intr_handle.vfio_dev_fd = vfio_dev_fd;\n+\tdev->req_notifier_handler.vfio_dev_fd = vfio_dev_fd;\n \n \treturn 0;\n err_vfio_dev_fd:\n@@ -658,6 +763,12 @@ pci_vfio_unmap_resource_primary(struct rte_pci_device *dev)\n \tsnprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,\n \t\t\tloc->domain, loc->bus, loc->devid, loc->function);\n \n+\tret = pci_vfio_disable_notifier(dev);\n+\tif (ret) {\n+\t\tRTE_LOG(ERR, EAL, \"fail to disable req notifier.\\n\");\n+\t\treturn -1;\n+\t}\n+\n \tif (close(dev->intr_handle.fd) < 0) {\n \t\tRTE_LOG(INFO, EAL, \"Error when closing eventfd file descriptor for %s\\n\",\n \t\t\tpci_addr);\ndiff --git a/drivers/bus/pci/pci_common.c b/drivers/bus/pci/pci_common.c\nindex f313fe9..51e69c1 100644\n--- a/drivers/bus/pci/pci_common.c\n+++ b/drivers/bus/pci/pci_common.c\n@@ -446,6 +446,16 @@ pci_hot_unplug_handler(struct rte_device *dev)\n \t\treturn -1;\n \n \tswitch (pdev->kdrv) {\n+\tcase RTE_KDRV_VFIO:\n+\t\t/*\n+\t\t * vfio kernel module guaranty the pci device would not be\n+\t\t * deleted until the user space release the resource, so no\n+\t\t * need to remap BARs resource here, just directly notify\n+\t\t * the req event to the user space to handle it.\n+\t\t */\n+\t\trte_dev_event_callback_process(dev->name,\n+\t\t\t\t\t RTE_DEV_EVENT_REMOVE);\n+\t\tbreak;\n \tcase RTE_KDRV_IGB_UIO:\n \tcase RTE_KDRV_UIO_GENERIC:\n \tcase RTE_KDRV_NIC_UIO:\n", "prefixes": [ "v3", "4/4" ] }