From patchwork Mon Sep 14 13:09:44 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?b?6LCi5Y2O5LyfKOatpOaXtuatpOWIu++8iQ==?= X-Patchwork-Id: 77608 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1DD2EA04C7; Mon, 14 Sep 2020 15:12:41 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 0368F2BAB; Mon, 14 Sep 2020 15:12:41 +0200 (CEST) Received: from out0-147.mail.aliyun.com (out0-147.mail.aliyun.com [140.205.0.147]) by dpdk.org (Postfix) with ESMTP id 14FD62BAB for ; Mon, 14 Sep 2020 15:09:52 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=alibaba-inc.com; s=default; t=1600088988; h=To:From:Subject:Message-ID:Date:MIME-Version:Content-Type; bh=GKXSYAu+n2yKe65zZrssnS8AUiEypTnYxPJXzmo4KTI=; b=T3xhiuq4gPHFvTfzneC/VmfzsPFPfuSu4TF6y1cG/Xsouic5UkJmYAnvX3XStnEkTnTzHirG/b9QNFOfdyZ0YpiJZx6ra9SEuYE1hr7DjKerfLChl86daoP9rJh+TkBX/ZjUcYAf8x1PJt5ATPAEAlA5wSAZIG5ss7q9EHQzld0= X-Alimail-AntiSpam: AC=PASS; BC=-1|-1; BR=01201311R131e4; CH=green; DM=||false|; DS=||; FP=0|-1|-1|-1|0|-1|-1|-1; HT=e02c03302; MF=huawei.xhw@alibaba-inc.com; NM=1; PH=DS; RN=4; SR=0; TI=SMTPD_---.IWz7BvS_1600088987; Received: from 30.43.67.177(mailfrom:huawei.xhw@alibaba-inc.com fp:SMTPD_---.IWz7BvS_1600088987) by smtp.aliyun-inc.com(127.0.0.1); Mon, 14 Sep 2020 21:09:47 +0800 To: dev@dpdk.org Cc: ferruh.yigit@intel.com, zhihong.wang@intel.com, chenbo.xia@intel.com From: " =?utf-8?b?6LCi5Y2O5LyfKOatpOaXtuatpOWIu++8iQ==?= " Message-ID: <7558efff-25ff-836b-12e1-140f34869ba5@alibaba-inc.com> Date: Mon, 14 Sep 2020 21:09:44 +0800 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Thunderbird/78.2.2 MIME-Version: 1.0 Content-Language: en-US Subject: [dpdk-dev] [PATCH] pci: support both PIO and MMIO bar for legacy virtio on x86 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" In previous implementation, with igb_uio we get PIO address from igb_uio sysfs entry; with uio_pci_generic, we get PIO address from /proc/ioports. For PIO/MMIO RW, there is different path for different drivers and arch. For VFIO, PIO port RW is through syscall, which has big performance issue. On X86, it assumes only PIO is supported. This is too much twisted. we unify the way to get both PIO and MMIO address for diffrent driver and arch, all from standard resource attr under pci sysfs. only configure space rw should go through driver specific method. Signed-off-by: huawei.xhw ---  drivers/bus/pci/linux/pci.c     |  77 +-------------------  drivers/bus/pci/linux/pci_uio.c | 156 +++++++++++++++++++++++++++-------------  2 files changed, 110 insertions(+), 123 deletions(-)          snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num);          dev->intr_handle.fd = open(filename, O_RDWR); @@ -413,11 +437,14 @@          dev->intr_handle.type = RTE_INTR_HANDLE_UIO;      } -    RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%lx\n", start); +    RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%lx\n", base); -    p->base = start; +    p->base = base;      p->len = 0;      return 0; +error: +    fclose(f); +    return -1;  }  #else  int @@ -488,6 +515,61 @@  }  #endif +#define PIO_MAX 0x10000 +static inline uint8_t ioread8(void *addr) +{ +    uint8_t val; + +    val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? +        *(volatile uint8_t *)addr : +        inb((unsigned long)addr); + +    return val; +} + +static inline uint16_t ioread16(void *addr) +{ +    uint16_t val; + +    val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? +        *(volatile uint16_t *)addr : +        inw((unsigned long)addr); + +    return val; +} + +static inline uint32_t ioread32(void *addr) +{ +    uint32_t val; + +    val = (uint64_t)(uintptr_t)addr >= PIO_MAX ? +        *(volatile uint32_t *)addr : +        inl((unsigned long)addr); + +    return val; +} + +static inline void iowrite8(uint8_t val, void *addr) +{ +    (uint64_t)(uintptr_t)addr >= PIO_MAX ? +        *(volatile uint8_t *)addr = val : +        outb(val, (unsigned long)addr); +} + +static inline void iowrite16(uint16_t val, void *addr) +{ +    (uint64_t)(uintptr_t)addr >= PIO_MAX ? +        *(volatile uint16_t *)addr = val : +        outw(val, (unsigned long)addr); +} + +static inline void iowrite32(uint32_t val, void *addr) +{ +    (uint64_t)(uintptr_t)addr >= PIO_MAX ? +        *(volatile uint32_t *)addr = val : +        outl(val, (unsigned long)addr); +} +  void  pci_uio_ioport_read(struct rte_pci_ioport *p,              void *data, size_t len, off_t offset) @@ -499,25 +581,13 @@      for (d = data; len > 0; d += size, reg += size, len -= size) {          if (len >= 4) {              size = 4; -#if defined(RTE_ARCH_X86) -            *(uint32_t *)d = inl(reg); -#else -            *(uint32_t *)d = *(volatile uint32_t *)reg; -#endif +            *(uint32_t *)d = ioread32((void *)reg);          } else if (len >= 2) {              size = 2; -#if defined(RTE_ARCH_X86) -            *(uint16_t *)d = inw(reg); -#else -            *(uint16_t *)d = *(volatile uint16_t *)reg; -#endif +            *(uint16_t *)d = ioread16((void *)reg);          } else {              size = 1; -#if defined(RTE_ARCH_X86) -            *d = inb(reg); -#else -            *d = *(volatile uint8_t *)reg; -#endif +            *d = ioread8((void *)reg);          }      }  } @@ -533,25 +603,13 @@      for (s = data; len > 0; s += size, reg += size, len -= size) {          if (len >= 4) {              size = 4; -#if defined(RTE_ARCH_X86) -            outl_p(*(const uint32_t *)s, reg); -#else -            *(volatile uint32_t *)reg = *(const uint32_t *)s; -#endif +            iowrite32(*(const uint32_t *)s, (void *)reg);          } else if (len >= 2) {              size = 2; -#if defined(RTE_ARCH_X86) -            outw_p(*(const uint16_t *)s, reg); -#else -            *(volatile uint16_t *)reg = *(const uint16_t *)s; -#endif +            iowrite16(*(const uint16_t *)s, (void *)reg);          } else {              size = 1; -#if defined(RTE_ARCH_X86) -            outb_p(*s, reg); -#else -            *(volatile uint8_t *)reg = *s; -#endif +            iowrite8(*s, (void *)reg);          }      }  } diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index a2198ab..7266001 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -687,71 +687,6 @@ int rte_pci_write_config(const struct rte_pci_device *device,      }  } -#if defined(RTE_ARCH_X86) -static int -pci_ioport_map(struct rte_pci_device *dev, int bar __rte_unused, -        struct rte_pci_ioport *p) -{ -    uint16_t start, end; -    FILE *fp; -    char *line = NULL; -    char pci_id[16]; -    int found = 0; -    size_t linesz; - -    if (rte_eal_iopl_init() != 0) { -        RTE_LOG(ERR, EAL, "%s(): insufficient ioport permissions for PCI device %s\n", -            __func__, dev->name); -        return -1; -    } - -    snprintf(pci_id, sizeof(pci_id), PCI_PRI_FMT, -         dev->addr.domain, dev->addr.bus, -         dev->addr.devid, dev->addr.function); - -    fp = fopen("/proc/ioports", "r"); -    if (fp == NULL) { -        RTE_LOG(ERR, EAL, "%s(): can't open ioports\n", __func__); -        return -1; -    } - -    while (getdelim(&line, &linesz, '\n', fp) > 0) { -        char *ptr = line; -        char *left; -        int n; - -        n = strcspn(ptr, ":"); -        ptr[n] = 0; -        left = &ptr[n + 1]; - -        while (*left && isspace(*left)) -            left++; - -        if (!strncmp(left, pci_id, strlen(pci_id))) { -            found = 1; - -            while (*ptr && isspace(*ptr)) -                ptr++; - -            sscanf(ptr, "%04hx-%04hx", &start, &end); - -            break; -        } -    } - -    free(line); -    fclose(fp); - -    if (!found) -        return -1; - -    p->base = start; -    RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%x\n", start); - -    return 0; -} -#endif -  int  rte_pci_ioport_map(struct rte_pci_device *dev, int bar,          struct rte_pci_ioport *p) @@ -762,18 +697,12 @@ int rte_pci_write_config(const struct rte_pci_device *device,  #ifdef VFIO_PRESENT      case RTE_KDRV_VFIO:          if (pci_vfio_is_enabled()) -            ret = pci_vfio_ioport_map(dev, bar, p); +            ret = pci_uio_ioport_map(dev, bar, p);          break;  #endif      case RTE_KDRV_IGB_UIO: -        ret = pci_uio_ioport_map(dev, bar, p); -        break;      case RTE_KDRV_UIO_GENERIC: -#if defined(RTE_ARCH_X86) -        ret = pci_ioport_map(dev, bar, p); -#else          ret = pci_uio_ioport_map(dev, bar, p); -#endif          break;      default:          break; @@ -792,7 +721,7 @@ int rte_pci_write_config(const struct rte_pci_device *device,      switch (p->dev->kdrv) {  #ifdef VFIO_PRESENT      case RTE_KDRV_VFIO: -        pci_vfio_ioport_read(p, data, len, offset); +        pci_uio_ioport_read(p, data, len, offset);          break;  #endif      case RTE_KDRV_IGB_UIO: @@ -813,7 +742,7 @@ int rte_pci_write_config(const struct rte_pci_device *device,      switch (p->dev->kdrv) {  #ifdef VFIO_PRESENT      case RTE_KDRV_VFIO: -        pci_vfio_ioport_write(p, data, len, offset); +        pci_uio_ioport_write(p, data, len, offset);          break;  #endif      case RTE_KDRV_IGB_UIO: diff --git a/drivers/bus/pci/linux/pci_uio.c b/drivers/bus/pci/linux/pci_uio.c index 097dc19..0d7ee3c 100644 --- a/drivers/bus/pci/linux/pci_uio.c +++ b/drivers/bus/pci/linux/pci_uio.c @@ -372,36 +372,60 @@  pci_uio_ioport_map(struct rte_pci_device *dev, int bar,             struct rte_pci_ioport *p)  { +    FILE *f;      char dirname[PATH_MAX];      char filename[PATH_MAX]; +    char buf[BUFSIZ]; +    uint64_t phys_addr, end_addr, flags;      int uio_num; -    unsigned long start; +    unsigned long base; +    bool iobar; +    int i; -    if (rte_eal_iopl_init() != 0) { -        RTE_LOG(ERR, EAL, "%s(): insufficient ioport permissions for PCI device %s\n", -            __func__, dev->name); +    /* open and read addresses of the corresponding resource in sysfs */ +    snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource", +        rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, +        dev->addr.devid, dev->addr.function); +    f = fopen(filename, "r"); +    if (f == NULL) { +        RTE_LOG(ERR, EAL, "Cannot open sysfs resource: %s\n", +            strerror(errno));          return -1;      } +    for (i = 0; i < bar + 1; i++) { +        if (fgets(buf, sizeof(buf), f) == NULL) { +            RTE_LOG(ERR, EAL, "Cannot read sysfs resource\n"); +            goto error; +        } +    } +    if (pci_parse_one_sysfs_resource(buf, sizeof(buf), &phys_addr, +            &end_addr, &flags) < 0) +        goto error; -    uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 0); -    if (uio_num < 0) -        return -1; +    if (flags & IORESOURCE_IO) { +        iobar = 1; +        base = (unsigned long)phys_addr; +        RTE_LOG(INFO, EAL, "%s(): iobar %08lx detected\n", __func__, base); +    } else if (flags & IORESOURCE_MEM) { +        iobar = 0; +        base = (unsigned long)dev->mem_resource[bar].addr; +        RTE_LOG(INFO, EAL, "%s(): membar %08lx detected\n", __func__, base); +    } else { +        RTE_LOG(ERR, EAL, "%s(): unknown bar type\n", __func__); +        goto error; +    } -    /* get portio start */ -    snprintf(filename, sizeof(filename), -         "%s/portio/port%d/start", dirname, bar); -    if (eal_parse_sysfs_value(filename, &start) < 0) { -        RTE_LOG(ERR, EAL, "%s(): cannot parse portio start\n", -            __func__); +    if (iobar && rte_eal_iopl_init() != 0) { +        RTE_LOG(ERR, EAL, "%s(): insufficient ioport permissions for PCI device %s\n", +            __func__, dev->name);          return -1;      } -    /* ensure we don't get anything funny here, read/write will cast to -     * uin16_t */ -    if (start > UINT16_MAX) -        return -1;      /* FIXME only for primary process ? */      if (dev->intr_handle.type == RTE_INTR_HANDLE_UNKNOWN) { +        uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 0); +        if (uio_num < 0) +            return -1;