From patchwork Thu Feb 19 17:08:49 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Zhou, Danny" X-Patchwork-Id: 3504 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [IPv6:::1]) by dpdk.org (Postfix) with ESMTP id 392AFB6D8; Thu, 19 Feb 2015 18:09:05 +0100 (CET) Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by dpdk.org (Postfix) with ESMTP id 884B0B6D7 for ; Thu, 19 Feb 2015 18:09:03 +0100 (CET) Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by orsmga101.jf.intel.com with ESMTP; 19 Feb 2015 09:09:01 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.09,609,1418112000"; d="scan'208";a="668701742" Received: from shvmail01.sh.intel.com ([10.239.29.42]) by fmsmga001.fm.intel.com with ESMTP; 19 Feb 2015 09:09:00 -0800 Received: from shecgisg004.sh.intel.com (shecgisg004.sh.intel.com [10.239.29.89]) by shvmail01.sh.intel.com with ESMTP id t1JH8v5Q010235; Fri, 20 Feb 2015 01:08:57 +0800 Received: from shecgisg004.sh.intel.com (localhost [127.0.0.1]) by shecgisg004.sh.intel.com (8.13.6/8.13.6/SuSE Linux 0.8) with ESMTP id t1JH8tOA032270; Fri, 20 Feb 2015 01:08:57 +0800 Received: (from dyzhou@localhost) by shecgisg004.sh.intel.com (8.13.6/8.13.6/Submit) id t1JH8tPr032266; Fri, 20 Feb 2015 01:08:55 +0800 From: Zhou Danny To: dev@dpdk.org Date: Fri, 20 Feb 2015 01:08:49 +0800 Message-Id: <1424365731-32228-2-git-send-email-danny.zhou@intel.com> X-Mailer: git-send-email 1.7.4.1 In-Reply-To: <1424365731-32228-1-git-send-email-danny.zhou@intel.com> References: <1424365731-32228-1-git-send-email-danny.zhou@intel.com> Subject: [dpdk-dev] [PATCH v2 1/3] eal: enable uio_pci_generic support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" v2 changes: - Change variable name of kernel driver with precise comment. - Fix a union definition error in v1 patchset. - Move redefined macro IORESOURCE_MEM to rte_pci.h with comment. 1) Unify procedure to retrieve BAR resource mapping information. 2) Setup bus master bit in NIC's PCIe configuration space for uio_pci_generic. Signed-off-by: Danny Zhou Tested-by: Qun Wan --- lib/librte_eal/common/include/rte_pci.h | 4 + lib/librte_eal/linuxapp/eal/eal_pci.c | 4 +- lib/librte_eal/linuxapp/eal/eal_pci_uio.c | 204 +++++++++++---------- .../linuxapp/eal/include/exec-env/rte_interrupts.h | 8 +- 4 files changed, 122 insertions(+), 98 deletions(-) diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h index 66ed793..85f116f 100644 --- a/lib/librte_eal/common/include/rte_pci.h +++ b/lib/librte_eal/common/include/rte_pci.h @@ -104,6 +104,9 @@ extern struct pci_device_list pci_device_list; /**< Global list of PCI devices. /** Nb. of values in PCI resource format. */ #define PCI_RESOURCE_FMT_NVAL 3 +/** IO resource type: memory address space */ +#define IORESOURCE_MEM 0x00000200 + /** * A structure describing a PCI resource. */ @@ -148,6 +151,7 @@ struct rte_pci_device { struct rte_pci_id id; /**< PCI ID. */ struct rte_pci_resource mem_resource[PCI_MAX_RESOURCE]; /**< PCI Memory Resource */ struct rte_intr_handle intr_handle; /**< Interrupt handle */ + char kernel_driver_name[BUFSIZ]; /**< Kernel driver name */ const struct rte_pci_driver *driver; /**< Associated driver */ uint16_t max_vfs; /**< sriov enable if not zero */ int numa_node; /**< NUMA node connection */ diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c index 15db9c4..63bcbce 100644 --- a/lib/librte_eal/linuxapp/eal/eal_pci.c +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c @@ -138,8 +138,6 @@ pci_map_resource(void *requested_addr, int fd, off_t offset, size_t size) } /* parse the "resource" sysfs file */ -#define IORESOURCE_MEM 0x00000200 - static int pci_parse_sysfs_resource(const char *filename, struct rte_pci_device *dev) { @@ -519,7 +517,7 @@ pci_map_device(struct rte_pci_device *dev) return ret; } #endif - /* map resources for devices that use igb_uio */ + /* map resources for devices that use uio_pci_generic or igb_uio */ if (!mapped) { ret = pci_uio_map_resource(dev); if (ret != 0) diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c index e53f06b..03ac006 100644 --- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c +++ b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -47,71 +48,72 @@ #include "eal_filesystem.h" #include "eal_pci_init.h" -static int pci_parse_sysfs_value(const char *filename, uint64_t *val); +/* driver name of uio_pci_generic kernel module */ +#define UIO_PCI_GENERIC_DRIVER_NAME "uio_pci_generic" void *pci_map_addr = NULL; #define OFF_MAX ((uint64_t)(off_t)-1) static int -pci_uio_get_mappings(const char *devname, struct pci_map maps[], int nb_maps) +pci_uio_get_mappings(struct rte_pci_device *dev, + struct pci_map maps[], int nb_maps) { - int i; - char dirname[PATH_MAX]; + struct rte_pci_addr *loc = &dev->addr; + int i = 0; char filename[PATH_MAX]; - uint64_t offset, size; - - for (i = 0; i != nb_maps; i++) { + unsigned long long start_addr, end_addr, flags; + FILE *f; - /* check if map directory exists */ - snprintf(dirname, sizeof(dirname), - "%s/maps/map%u", devname, i); + snprintf(filename, sizeof(filename), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/resource", + loc->domain, loc->bus, loc->devid, loc->function); - if (access(dirname, F_OK) != 0) - break; + f = fopen(filename, "r"); + if (f == NULL) { + RTE_LOG(ERR, EAL, + "%s(): cannot open sysfs %s\n", + __func__, filename); + return -1; + } - /* get mapping offset */ - snprintf(filename, sizeof(filename), - "%s/offset", dirname); - if (pci_parse_sysfs_value(filename, &offset) < 0) { - RTE_LOG(ERR, EAL, - "%s(): cannot parse offset of %s\n", - __func__, dirname); - return -1; + while (fscanf(f, "%llx %llx %llx", &start_addr, + &end_addr, &flags) == 3 && i < nb_maps) { + if (flags & IORESOURCE_MEM) { + maps[i].offset = 0x0; + maps[i].size = end_addr - start_addr + 1; + maps[i].phaddr = start_addr; + i++; } + } + fclose(f); - /* get mapping size */ - snprintf(filename, sizeof(filename), - "%s/size", dirname); - if (pci_parse_sysfs_value(filename, &size) < 0) { - RTE_LOG(ERR, EAL, - "%s(): cannot parse size of %s\n", - __func__, dirname); - return -1; - } + return i; +} - /* get mapping physical address */ - snprintf(filename, sizeof(filename), - "%s/addr", dirname); - if (pci_parse_sysfs_value(filename, &maps[i].phaddr) < 0) { - RTE_LOG(ERR, EAL, - "%s(): cannot parse addr of %s\n", - __func__, dirname); - return -1; - } +static int +pci_uio_set_bus_master(int dev_fd) +{ + uint16_t reg; + int ret; - if ((offset > OFF_MAX) || (size > SIZE_MAX)) { - RTE_LOG(ERR, EAL, - "%s(): offset/size exceed system max value\n", - __func__); - return -1; - } + ret = pread(dev_fd, ®, sizeof(reg), PCI_COMMAND); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, + "Cannot read command from PCI config space!\n"); + return -1; + } + + reg |= PCI_COMMAND_MASTER; - maps[i].offset = offset; - maps[i].size = size; + ret = pwrite(dev_fd, ®, sizeof(reg), PCI_COMMAND); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, + "Cannot write command to PCI config space!\n"); + return -1; } - return i; + return 0; } static int @@ -213,6 +215,10 @@ pci_get_uio_dev(struct rte_pci_device *dev, char *dstbuf, struct dirent *e; DIR *dir; char dirname[PATH_MAX]; + FILE *f; + char fullpath[PATH_MAX]; + char buf[BUFSIZ]; + char *s; /* depending on kernel version, uio can be located in uio/uioX * or uio:uioX */ @@ -268,6 +274,30 @@ pci_get_uio_dev(struct rte_pci_device *dev, char *dstbuf, if (e == NULL) return -1; + /* remember driver name of uio device */ + snprintf(fullpath, sizeof(fullpath), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/uio/%s/name" , + loc->domain, loc->bus, loc->devid, + loc->function, e->d_name); + + f = fopen(fullpath, "r"); + if (f == NULL) { + RTE_LOG(ERR, EAL, + "%s(): cannot open sysfs to get driver name\n", + __func__); + return -1; + } + s = fgets(buf, sizeof(buf), f); + if (s == NULL) { + RTE_LOG(ERR, EAL, + "%s(): cannot read sysfs to get driver name\n", + __func__); + fclose(f); + return -1; + } + strncpy(dev->kernel_driver_name, buf, sizeof(buf)); + fclose(f); + /* create uio device if we've been asked to */ if (internal_config.create_uio_dev && pci_mknod_uio_dev(dstbuf, uio_num) < 0) @@ -282,6 +312,7 @@ pci_uio_map_resource(struct rte_pci_device *dev) { int i, j; char dirname[PATH_MAX]; + char cfgname[PATH_MAX]; char devname[PATH_MAX]; /* contains the /dev/uioX */ void *mapaddr; int uio_num; @@ -294,6 +325,7 @@ pci_uio_map_resource(struct rte_pci_device *dev) struct pci_map *maps; dev->intr_handle.fd = -1; + dev->intr_handle.uio_cfg_fd = -1; dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; /* secondary processes - use already recorded details */ @@ -318,6 +350,33 @@ pci_uio_map_resource(struct rte_pci_device *dev) } dev->intr_handle.type = RTE_INTR_HANDLE_UIO; + snprintf(cfgname, sizeof(cfgname), + "/sys/class/uio/uio%u/device/config", uio_num); + dev->intr_handle.uio_cfg_fd = open(cfgname, O_RDWR); + if (dev->intr_handle.uio_cfg_fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + cfgname, strerror(errno)); + return -1; + } + + /* handle Misc. stuff for uio_pci_generic */ + if (strncmp(dev->kernel_driver_name, UIO_PCI_GENERIC_DRIVER_NAME, + strlen(UIO_PCI_GENERIC_DRIVER_NAME)) == 0) { + /* update devname for uio_pci_generic */ + snprintf(devname, sizeof(devname), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/resource%d", + loc->domain, loc->bus, loc->devid, loc->function, 0); + + /* set bus master that is not done by uio_pci_generic */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + if (pci_uio_set_bus_master(dev->intr_handle.uio_cfg_fd)) { + RTE_LOG(ERR, EAL, + "Cannot set up bus mastering!\n"); + return -1; + } + } + } + /* allocate the mapping details for secondary processes*/ uio_res = rte_zmalloc("UIO_RES", sizeof(*uio_res), 0); if (uio_res == NULL) { @@ -330,13 +389,12 @@ pci_uio_map_resource(struct rte_pci_device *dev) memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr)); /* collect info about device mappings */ - nb_maps = pci_uio_get_mappings(dirname, uio_res->maps, - RTE_DIM(uio_res->maps)); + nb_maps = pci_uio_get_mappings(dev, uio_res->maps, + RTE_DIM(uio_res->maps)); if (nb_maps < 0) { rte_free(uio_res); return nb_maps; } - uio_res->nb_maps = nb_maps; /* Map all BARs */ @@ -374,16 +432,11 @@ pci_uio_map_resource(struct rte_pci_device *dev) if (maps[j].addr != NULL) fail = 1; else { - /* try mapping somewhere close to the end of hugepages */ - if (pci_map_addr == NULL) - pci_map_addr = pci_find_max_end_va(); - - mapaddr = pci_map_resource(pci_map_addr, fd, (off_t)offset, + mapaddr = pci_map_resource(NULL, fd, + (off_t)offset, (size_t)maps[j].size); - if (mapaddr == MAP_FAILED) + if (mapaddr == NULL) fail = 1; - - pci_map_addr = RTE_PTR_ADD(mapaddr, (size_t) maps[j].size); } if (fail) { @@ -403,38 +456,3 @@ pci_uio_map_resource(struct rte_pci_device *dev) return 0; } - -/* - * parse a sysfs file containing one integer value - * different to the eal version, as it needs to work with 64-bit values - */ -static int -pci_parse_sysfs_value(const char *filename, uint64_t *val) -{ - FILE *f; - char buf[BUFSIZ]; - char *end = NULL; - - f = fopen(filename, "r"); - if (f == NULL) { - RTE_LOG(ERR, EAL, "%s(): cannot open sysfs value %s\n", - __func__, filename); - return -1; - } - - if (fgets(buf, sizeof(buf), f) == NULL) { - RTE_LOG(ERR, EAL, "%s(): cannot read sysfs value %s\n", - __func__, filename); - fclose(f); - return -1; - } - *val = strtoull(buf, &end, 0); - if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) { - RTE_LOG(ERR, EAL, "%s(): cannot parse sysfs value %s\n", - __func__, filename); - fclose(f); - return -1; - } - fclose(f); - return 0; -} diff --git a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h index 23eafd9..6a159c7 100644 --- a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h +++ b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h @@ -50,8 +50,12 @@ enum rte_intr_handle_type { /** Handle for interrupts. */ struct rte_intr_handle { - int vfio_dev_fd; /**< VFIO device file descriptor */ - int fd; /**< file descriptor */ + union { + int vfio_dev_fd; /**< VFIO device file descriptor */ + int uio_cfg_fd; /**< UIO config file descriptor + for uio_pci_generic */ + }; + int fd; /**< interrupt event file descriptor */ enum rte_intr_handle_type type; /**< handle type */ };