diff mbox

[dpdk-dev,v7] eal: map PCI memory resources after hugepages

Message ID 1415619272-8281-1-git-send-email-anatoly.burakov@intel.com (mailing list archive)
State Superseded, archived
Headers show

Commit Message

Burakov, Anatoly Nov. 10, 2014, 11:34 a.m. UTC
Multi-process DPDK application must mmap hugepages and pci resources
into the same virtual address space. By default the virtual addresses
are chosen by the primary process automatically when calling the mmap.
But sometimes the chosen virtual addresses aren't usable in secondary
process - for example, secondary process is linked with more libraries
than primary process, and the library occupies the same address space
that the primary process has requested for PCI mappings.

This patch makes EAL map PCI BARs right after the hugepages (instead of
location chosen by mmap) in virtual memory.

Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
Signed-off-by: Liang Xu <liang.xu@cinfotech.cn>
---
 lib/librte_eal/linuxapp/eal/eal_pci.c              | 19 +++++++++++++++++++
 lib/librte_eal/linuxapp/eal/eal_pci_uio.c          |  9 ++++++++-
 lib/librte_eal/linuxapp/eal/eal_pci_vfio.c         | 13 +++++++++++--
 lib/librte_eal/linuxapp/eal/include/eal_pci_init.h |  6 ++++++
 4 files changed, 44 insertions(+), 3 deletions(-)
diff mbox

Patch

diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
index 5fe3961..dae8739 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
@@ -97,6 +97,25 @@  error:
 	return -1;
 }
 
+void *
+pci_find_max_end_va(void)
+{
+	const struct rte_memseg *seg = rte_eal_get_physmem_layout();
+	const struct rte_memseg *last = seg;
+	unsigned i = 0;
+
+	for (i = 0; i < RTE_MAX_MEMSEG; i++, seg++) {
+		if (seg->addr == NULL)
+			break;
+
+		if (seg->addr > last->addr)
+			last = seg;
+
+	}
+	return RTE_PTR_ADD(last->addr, last->len);
+}
+
+
 /* map a particular resource from a file */
 void *
 pci_map_resource(void *requested_addr, int fd, off_t offset, size_t size)
diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
index 7e62266..5090bf1 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
@@ -48,6 +48,8 @@ 
 
 static int pci_parse_sysfs_value(const char *filename, uint64_t *val);
 
+void *pci_map_addr = NULL;
+
 
 #define OFF_MAX              ((uint64_t)(off_t)-1)
 static int
@@ -371,10 +373,15 @@  pci_uio_map_resource(struct rte_pci_device *dev)
 			if (maps[j].addr != NULL)
 				fail = 1;
 			else {
-				mapaddr = pci_map_resource(NULL, fd, (off_t)offset,
+				if (pci_map_addr == NULL)
+					pci_map_addr = pci_find_max_end_va();
+
+				mapaddr = pci_map_resource(pci_map_addr, fd, (off_t)offset,
 						(size_t)maps[j].size);
 				if (mapaddr == NULL)
 					fail = 1;
+
+				pci_map_addr = RTE_PTR_ADD(pci_map_addr, maps[j].size);
 			}
 
 			if (fail) {
diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c b/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c
index c776ddc..fb6ee7a 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c
@@ -720,8 +720,17 @@  pci_vfio_map_resource(struct rte_pci_device *dev)
 		if (i == msix_bar)
 			continue;
 
-		bar_addr = pci_map_resource(maps[i].addr, vfio_dev_fd, reg.offset,
-				reg.size);
+		if (internal_config.process_type == RTE_PROC_PRIMARY) {
+			if (pci_map_addr == NULL)
+				pci_map_addr = pci_find_max_end_va();
+
+			bar_addr = pci_map_resource(pci_map_addr, vfio_dev_fd, reg.offset,
+					reg.size);
+			pci_map_addr = RTE_PTR_ADD(pci_map_addr, reg.size);
+		} else {
+			bar_addr = pci_map_resource(maps[i].addr, vfio_dev_fd, reg.offset,
+					reg.size);
+		}
 
 		if (bar_addr == NULL) {
 			RTE_LOG(ERR, EAL, "  %s mapping BAR%i failed: %s\n", pci_addr, i,
diff --git a/lib/librte_eal/linuxapp/eal/include/eal_pci_init.h b/lib/librte_eal/linuxapp/eal/include/eal_pci_init.h
index d758bee..1070eb8 100644
--- a/lib/librte_eal/linuxapp/eal/include/eal_pci_init.h
+++ b/lib/librte_eal/linuxapp/eal/include/eal_pci_init.h
@@ -59,6 +59,12 @@  struct mapped_pci_resource {
 TAILQ_HEAD(mapped_pci_res_list, mapped_pci_resource);
 extern struct mapped_pci_res_list *pci_res_list;
 
+/*
+ * Helper function to map PCI resources right after hugepages in virtual memory
+ */
+extern void *pci_map_addr;
+void *pci_find_max_end_va(void);
+
 void *pci_map_resource(void *requested_addr, int fd, off_t offset,
 		size_t size);