From patchwork Thu Jul 12 14:01:43 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Qi Zhang X-Patchwork-Id: 42976 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id EB7381B472; Thu, 12 Jul 2018 16:01:16 +0200 (CEST) Received: from mga04.intel.com (mga04.intel.com [192.55.52.120]) by dpdk.org (Postfix) with ESMTP id 0C0831B448 for ; Thu, 12 Jul 2018 16:01:10 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga008.jf.intel.com ([10.7.209.65]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 12 Jul 2018 07:01:10 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.51,343,1526367600"; d="scan'208";a="56361081" Received: from dpdk51.sh.intel.com ([10.67.110.190]) by orsmga008.jf.intel.com with ESMTP; 12 Jul 2018 07:01:08 -0700 From: Qi Zhang To: thomas@monjalon.net, anatoly.burakov@intel.com, gaetan.rivet@6wind.com Cc: dev@dpdk.org, Qi Zhang Date: Thu, 12 Jul 2018 22:01:43 +0800 Message-Id: <20180712140144.18146-4-qi.z.zhang@intel.com> X-Mailer: git-send-email 2.13.6 In-Reply-To: <20180712140144.18146-1-qi.z.zhang@intel.com> References: <20180712140144.18146-1-qi.z.zhang@intel.com> Subject: [dpdk-dev] [PATCH 3/4] bus/pci: enable vfio unmap resource for secondary X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Subroutine to unmap VFIO resource is shared by secondary and primary, and it does not work on the secondary process. Since for secondary process, it is not necessary to close interrupt handler, set pci bus mastering and remove vfio_res from vfio_res_list. So, the patch adds a dedicate function to handle the situation when a device is unmapped on a secondary process. Signed-off-by: Qi Zhang Reviewed-by: Anatoly Burakov --- drivers/bus/pci/linux/pci_vfio.c | 118 +++++++++++++++++++++++++++++---------- 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c index 933b95540..686386d6a 100644 --- a/drivers/bus/pci/linux/pci_vfio.c +++ b/drivers/bus/pci/linux/pci_vfio.c @@ -584,6 +584,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev) dev->mem_resource[i].addr = maps[i].addr; } + /* we need save vfio_dev_fd, so it can be used during release */ + dev->intr_handle.vfio_dev_fd = vfio_dev_fd; + return 0; err_vfio_dev_fd: close(vfio_dev_fd); @@ -603,22 +606,58 @@ pci_vfio_map_resource(struct rte_pci_device *dev) return pci_vfio_map_resource_secondary(dev); } -int -pci_vfio_unmap_resource(struct rte_pci_device *dev) +static struct mapped_pci_resource * +find_and_unmap_vfio_resource(struct mapped_pci_res_list *vfio_res_list, + struct rte_pci_device *dev, + const char *pci_addr) +{ + struct mapped_pci_resource *vfio_res = NULL; + struct pci_map *maps; + int i; + + /* Get vfio_res */ + TAILQ_FOREACH(vfio_res, vfio_res_list, next) { + if (rte_pci_addr_cmp(&vfio_res->pci_addr, &dev->addr)) + continue; + break; + } + + if (vfio_res == NULL) + return vfio_res; + + RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n", + pci_addr); + + maps = vfio_res->maps; + for (i = 0; i < (int) vfio_res->nb_maps; i++) { + + /* + * We do not need to be aware of MSI-X table BAR mappings as + * when mapping. Just using current maps array is enough + */ + if (maps[i].addr) { + RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n", + pci_addr, maps[i].addr); + pci_unmap_resource(maps[i].addr, maps[i].size); + } + } + + return vfio_res; +} + +static int +pci_vfio_unmap_resource_primary(struct rte_pci_device *dev) { char pci_addr[PATH_MAX] = {0}; struct rte_pci_addr *loc = &dev->addr; - int i, ret; struct mapped_pci_resource *vfio_res = NULL; struct mapped_pci_res_list *vfio_res_list; - - struct pci_map *maps; + int ret; /* store PCI address string */ snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, loc->domain, loc->bus, loc->devid, loc->function); - if (close(dev->intr_handle.fd) < 0) { RTE_LOG(INFO, EAL, "Error when closing eventfd file descriptor for %s\n", pci_addr); @@ -639,13 +678,10 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev) return ret; } - vfio_res_list = RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); - /* Get vfio_res */ - TAILQ_FOREACH(vfio_res, vfio_res_list, next) { - if (rte_pci_addr_cmp(&vfio_res->pci_addr, &dev->addr)) - continue; - break; - } + vfio_res_list = + RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); + vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr); + /* if we haven't found our tailq entry, something's wrong */ if (vfio_res == NULL) { RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n", @@ -653,30 +689,56 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev) return -1; } - /* unmap BARs */ - maps = vfio_res->maps; + TAILQ_REMOVE(vfio_res_list, vfio_res, next); - RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n", - pci_addr); - for (i = 0; i < (int) vfio_res->nb_maps; i++) { + return 0; +} - /* - * We do not need to be aware of MSI-X table BAR mappings as - * when mapping. Just using current maps array is enough - */ - if (maps[i].addr) { - RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n", - pci_addr, maps[i].addr); - pci_unmap_resource(maps[i].addr, maps[i].size); - } +static int +pci_vfio_unmap_resource_secondary(struct rte_pci_device *dev) +{ + char pci_addr[PATH_MAX] = {0}; + struct rte_pci_addr *loc = &dev->addr; + struct mapped_pci_resource *vfio_res = NULL; + struct mapped_pci_res_list *vfio_res_list; + int ret; + + /* store PCI address string */ + snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, + loc->domain, loc->bus, loc->devid, loc->function); + + ret = rte_vfio_release_device(rte_pci_get_sysfs_path(), pci_addr, + dev->intr_handle.vfio_dev_fd); + if (ret < 0) { + RTE_LOG(ERR, EAL, + "%s(): cannot release device\n", __func__); + return ret; } - TAILQ_REMOVE(vfio_res_list, vfio_res, next); + vfio_res_list = + RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); + vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr); + + /* if we haven't found our tailq entry, something's wrong */ + if (vfio_res == NULL) { + RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n", + pci_addr); + return -1; + } return 0; } int +pci_vfio_unmap_resource(struct rte_pci_device *dev) +{ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + return pci_vfio_unmap_resource_primary(dev); + else + return pci_vfio_unmap_resource_secondary(dev); +} + +int pci_vfio_ioport_map(struct rte_pci_device *dev, int bar, struct rte_pci_ioport *p) {