[dpdk-dev,5/7] eal/linux: mmap ioports on ppc64

Message ID 1463143859-3105-6-git-send-email-olivier.matz@6wind.com (mailing list archive)
State Superseded, archived
Delegated to: Thomas Monjalon
Headers

Commit Message

Olivier Matz May 13, 2016, 12:50 p.m. UTC
  On PPC64, the ioports are mapped in memory. Implement the missing part
of ioport API for PPC64 when using uio. This may also work on other
architectures but it has not been tested.

Signed-off-by: David Marchand <david.marchand@6wind.com>
Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
---
 lib/librte_eal/common/include/rte_pci.h    |   4 +-
 lib/librte_eal/linuxapp/eal/eal_pci.c      |   4 +-
 lib/librte_eal/linuxapp/eal/eal_pci_init.h |   6 ++
 lib/librte_eal/linuxapp/eal/eal_pci_uio.c  | 119 +++++++++++++++++++++++------
 4 files changed, 107 insertions(+), 26 deletions(-)
  

Comments

David Marchand May 13, 2016, 2:33 p.m. UTC | #1
On Fri, May 13, 2016 at 2:50 PM, Olivier Matz <olivier.matz@6wind.com> wrote:
> On PPC64, the ioports are mapped in memory. Implement the missing part
> of ioport API for PPC64 when using uio. This may also work on other
> architectures but it has not been tested.
>
> Signed-off-by: David Marchand <david.marchand@6wind.com>
> Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
> ---
>  lib/librte_eal/common/include/rte_pci.h    |   4 +-
>  lib/librte_eal/linuxapp/eal/eal_pci.c      |   4 +-
>  lib/librte_eal/linuxapp/eal/eal_pci_init.h |   6 ++
>  lib/librte_eal/linuxapp/eal/eal_pci_uio.c  | 119 +++++++++++++++++++++++------
>  4 files changed, 107 insertions(+), 26 deletions(-)
>
> diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
> index fd049d1..4657396 100644
> --- a/lib/librte_eal/common/include/rte_pci.h
> +++ b/lib/librte_eal/common/include/rte_pci.h
> @@ -105,7 +105,8 @@ extern struct pci_device_list pci_device_list; /**< Global list of PCI devices.
>  /** Nb. of values in PCI resource format. */
>  #define PCI_RESOURCE_FMT_NVAL 3
>
> -/** IO resource type: memory address space */
> +/** IO resource type: */
> +#define IORESOURCE_IO         0x00000100
>  #define IORESOURCE_MEM        0x00000200
>
>  /**

This could go in a linux-specific header.

> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
> index 1a93725..f1ea52b 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_pci.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
> @@ -191,9 +191,9 @@ pci_find_max_end_va(void)
>  }
>
>  /* parse one line of the "resource" sysfs file (note that the 'line'
> - * string is modified)
> + * string is modified
>   */

Garbage ?
  
Olivier Matz May 13, 2016, 4:36 p.m. UTC | #2
On 05/13/2016 04:33 PM, David Marchand wrote:
>> --- a/lib/librte_eal/common/include/rte_pci.h
>> +++ b/lib/librte_eal/common/include/rte_pci.h
>> @@ -105,7 +105,8 @@ extern struct pci_device_list pci_device_list; /**< Global list of PCI devices.
>>  /** Nb. of values in PCI resource format. */
>>  #define PCI_RESOURCE_FMT_NVAL 3
>>
>> -/** IO resource type: memory address space */
>> +/** IO resource type: */
>> +#define IORESOURCE_IO         0x00000100
>>  #define IORESOURCE_MEM        0x00000200
>>
>>  /**
> 
> This could go in a linux-specific header.

Yep, I'll do a separate patch for that in v2.

> 
>> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
>> index 1a93725..f1ea52b 100644
>> --- a/lib/librte_eal/linuxapp/eal/eal_pci.c
>> +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
>> @@ -191,9 +191,9 @@ pci_find_max_end_va(void)
>>  }
>>
>>  /* parse one line of the "resource" sysfs file (note that the 'line'
>> - * string is modified)
>> + * string is modified
>>   */
> 
> Garbage ?

Indeed :)


Thanks for reviewing!
  

Patch

diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
index fd049d1..4657396 100644
--- a/lib/librte_eal/common/include/rte_pci.h
+++ b/lib/librte_eal/common/include/rte_pci.h
@@ -105,7 +105,8 @@  extern struct pci_device_list pci_device_list; /**< Global list of PCI devices.
 /** Nb. of values in PCI resource format. */
 #define PCI_RESOURCE_FMT_NVAL 3
 
-/** IO resource type: memory address space */
+/** IO resource type: */
+#define IORESOURCE_IO         0x00000100
 #define IORESOURCE_MEM        0x00000200
 
 /**
@@ -518,6 +519,7 @@  int rte_eal_pci_write_config(const struct rte_pci_device *device,
 struct rte_pci_ioport {
 	struct rte_pci_device *dev;
 	uint64_t base;
+	uint64_t len; /* only filled for memory mapped ports */
 };
 
 /**
diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
index 1a93725..f1ea52b 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
@@ -191,9 +191,9 @@  pci_find_max_end_va(void)
 }
 
 /* parse one line of the "resource" sysfs file (note that the 'line'
- * string is modified)
+ * string is modified
  */
-static int
+int
 pci_parse_one_sysfs_resource(char *line, size_t len, uint64_t *phys_addr,
 	uint64_t *end_addr, uint64_t *flags)
 {
diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_init.h b/lib/librte_eal/linuxapp/eal/eal_pci_init.h
index 7011753..fbe34c6 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci_init.h
+++ b/lib/librte_eal/linuxapp/eal/eal_pci_init.h
@@ -42,6 +42,12 @@ 
 extern void *pci_map_addr;
 void *pci_find_max_end_va(void);
 
+/* parse one line of the "resource" sysfs file (note that the 'line'
+ * string is modified)
+ */
+int pci_parse_one_sysfs_resource(char *line, size_t len, uint64_t *phys_addr,
+	uint64_t *end_addr, uint64_t *flags);
+
 int pci_uio_alloc_resource(struct rte_pci_device *dev,
 		struct mapped_pci_resource **uio_res);
 void pci_uio_free_resource(struct rte_pci_device *dev,
diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
index ac449c5..077ad96 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
@@ -35,6 +35,7 @@ 
 #include <unistd.h>
 #include <fcntl.h>
 #include <dirent.h>
+#include <inttypes.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
 #include <linux/pci_regs.h>
@@ -368,11 +369,11 @@  error:
 	return -1;
 }
 
+#if defined(RTE_ARCH_X86)
 int
 pci_uio_ioport_map(struct rte_pci_device *dev, int bar,
 		   struct rte_pci_ioport *p)
 {
-#if defined(RTE_ARCH_X86)
 	char dirname[PATH_MAX];
 	char filename[PATH_MAX];
 	int uio_num;
@@ -411,81 +412,153 @@  pci_uio_ioport_map(struct rte_pci_device *dev, int bar,
 	RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%lx\n", start);
 
 	p->base = start;
+	p->len = 0;
 	return 0;
+}
 #else
-	RTE_SET_USED(dev);
-	RTE_SET_USED(bar);
-	RTE_SET_USED(p);
+int
+pci_uio_ioport_map(struct rte_pci_device *dev, int bar,
+		   struct rte_pci_ioport *p)
+{
+	FILE *f;
+	char buf[BUFSIZ];
+	char filename[PATH_MAX];
+	uint64_t phys_addr, end_addr, flags;
+	int fd, i;
+	void *addr;
+
+	/* open and read addresses of the corresponding resource in sysfs */
+	snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource",
+		SYSFS_PCI_DEVICES, dev->addr.domain, dev->addr.bus,
+		 dev->addr.devid, dev->addr.function);
+	f = fopen(filename, "r");
+	if (f == NULL) {
+		RTE_LOG(ERR, EAL, "Cannot open sysfs resource: %s\n",
+			strerror(errno));
+		return -1;
+	}
+	for (i = 0; i < bar + 1; i++) {
+		if (fgets(buf, sizeof(buf), f) == NULL) {
+			RTE_LOG(ERR, EAL, "Cannot read sysfs resource\n");
+			goto error;
+		}
+	}
+	if (pci_parse_one_sysfs_resource(buf, sizeof(buf), &phys_addr,
+			&end_addr, &flags) < 0)
+		goto error;
+	if ((flags & IORESOURCE_IO) == 0) {
+		RTE_LOG(ERR, EAL, "BAR %d is not an IO resource\n", bar);
+		goto error;
+	}
+	snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource%d",
+		SYSFS_PCI_DEVICES, dev->addr.domain, dev->addr.bus,
+		 dev->addr.devid, dev->addr.function, bar);
+
+	/* mmap the pci resource */
+	fd = open(filename, O_RDWR);
+	if (fd < 0) {
+		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", filename,
+			strerror(errno));
+		goto error;
+	}
+	addr = mmap(NULL, end_addr + 1, PROT_READ | PROT_WRITE,
+		MAP_SHARED, fd, 0);
+	if (addr == MAP_FAILED) {
+		RTE_LOG(ERR, EAL, "Cannot mmap IO port resource: %s\n",
+			strerror(errno));
+		goto error;
+	}
+
+	/* strangely, the base address is mmap addr + phys_addr */
+	p->base = (uintptr_t)addr + phys_addr;
+	p->len = end_addr + 1;
+	RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%"PRIx64"\n", p->base);
+	fclose(f);
+
+	return 0;
+
+error:
+	fclose(f);
 	return -1;
-#endif
 }
+#endif
 
 void
 pci_uio_ioport_read(struct rte_pci_ioport *p,
 		    void *data, size_t len, off_t offset)
 {
-#if defined(RTE_ARCH_X86)
 	uint8_t *d;
 	int size;
-	unsigned short reg = p->base + offset;
+	uintptr_t reg = p->base + offset;
 
 	for (d = data; len > 0; d += size, reg += size, len -= size) {
 		if (len >= 4) {
 			size = 4;
+#if defined(RTE_ARCH_X86)
 			*(uint32_t *)d = inl(reg);
+#else
+			*(uint32_t *)d = *(volatile uint32_t *)reg;
+#endif
 		} else if (len >= 2) {
 			size = 2;
+#if defined(RTE_ARCH_X86)
 			*(uint16_t *)d = inw(reg);
+#else
+			*(uint16_t *)d = *(volatile uint16_t *)reg;
+#endif
 		} else {
 			size = 1;
+#if defined(RTE_ARCH_X86)
 			*d = inb(reg);
-		}
-	}
 #else
-	RTE_SET_USED(p);
-	RTE_SET_USED(data);
-	RTE_SET_USED(len);
-	RTE_SET_USED(offset);
+			*d = *(volatile uint8_t *)reg;
 #endif
+		}
+	}
 }
 
 void
 pci_uio_ioport_write(struct rte_pci_ioport *p,
 		     const void *data, size_t len, off_t offset)
 {
-#if defined(RTE_ARCH_X86)
 	const uint8_t *s;
 	int size;
-	unsigned short reg = p->base + offset;
+	uintptr_t reg = p->base + offset;
 
 	for (s = data; len > 0; s += size, reg += size, len -= size) {
 		if (len >= 4) {
 			size = 4;
+#if defined(RTE_ARCH_X86)
 			outl_p(*(const uint32_t *)s, reg);
+#else
+			*(volatile uint32_t *)reg = *(const uint32_t *)s;
+#endif
 		} else if (len >= 2) {
 			size = 2;
+#if defined(RTE_ARCH_X86)
 			outw_p(*(const uint16_t *)s, reg);
+#else
+			*(volatile uint16_t *)reg = *(const uint16_t *)s;
+#endif
 		} else {
 			size = 1;
+#if defined(RTE_ARCH_X86)
 			outb_p(*s, reg);
-		}
-	}
 #else
-	RTE_SET_USED(p);
-	RTE_SET_USED(data);
-	RTE_SET_USED(len);
-	RTE_SET_USED(offset);
+			*(volatile uint8_t *)reg = *s;
 #endif
+		}
+	}
 }
 
 int
 pci_uio_ioport_unmap(struct rte_pci_ioport *p)
 {
-	RTE_SET_USED(p);
 #if defined(RTE_ARCH_X86)
+	RTE_SET_USED(p);
 	/* FIXME close intr fd ? */
 	return 0;
 #else
-	return -1;
+	return munmap((void *)(uintptr_t)p->base, p->len);
 #endif
 }