[v4,3/4] test: add test case to validate VFIO DMA map/unmap
diff mbox series

Message ID 20201202054647.3449-4-ndabilpuram@marvell.com
State Superseded
Delegated to: David Marchand
Headers show
Series
  • fix issue with partial DMA unmap
Related show

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Nithin Dabilpuram Dec. 2, 2020, 5:46 a.m. UTC
Test case mmap's system pages and tries to performs a user
DMA map and unmap both partially and fully.

Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
---
 app/test/meson.build |   1 +
 app/test/test_vfio.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 app/test/test_vfio.c

Comments

David Christensen Dec. 2, 2020, 7:23 p.m. UTC | #1
On 12/1/20 9:46 PM, Nithin Dabilpuram wrote:
> Test case mmap's system pages and tries to performs a user
> DMA map and unmap both partially and fully.
> 
> Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> ---
>   app/test/meson.build |   1 +
>   app/test/test_vfio.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++
>   2 files changed, 104 insertions(+)
>   create mode 100644 app/test/test_vfio.c
> 
> diff --git a/app/test/meson.build b/app/test/meson.build
> index 94fd39f..d9eedb6 100644
> --- a/app/test/meson.build
> +++ b/app/test/meson.build
> @@ -139,6 +139,7 @@ test_sources = files('commands.c',
>   	'test_trace_register.c',
>   	'test_trace_perf.c',
>   	'test_version.c',
> +	'test_vfio.c',
>   	'virtual_pmd.c'
>   )
> 
> diff --git a/app/test/test_vfio.c b/app/test/test_vfio.c
> new file mode 100644
> index 0000000..00626d4
> --- /dev/null
> +++ b/app/test/test_vfio.c
> @@ -0,0 +1,103 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(C) 2020 Marvell.
> + */
> +
> +#include <stdio.h>
> +#include <stdint.h>
> +#include <string.h>
> +#include <sys/mman.h>
> +#include <unistd.h>
> +
> +#include <rte_common.h>
> +#include <rte_eal.h>
> +#include <rte_eal_paging.h>
> +#include <rte_errno.h>
> +#include <rte_memory.h>
> +#include <rte_vfio.h>
> +
> +#include "test.h"
> +
> +static int
> +test_memory_vfio_dma_map(void)
> +{
> +	uint64_t sz1, sz2, sz = 2 * rte_mem_page_size();
> +	uint64_t unmap1, unmap2;
> +	uint8_t *mem;
> +	int ret;
> +
> +	/* Allocate twice size of page */
> +	mem = mmap(NULL, sz, PROT_READ | PROT_WRITE,
> +		   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> +	if (mem == MAP_FAILED) {
> +		printf("Failed to allocate memory for external heap\n");
> +		return -1;
> +	}
> +
> +	/* Force page allocation */
> +	memset(mem, 0, sz);
> +
> +	/* map the whole region */
> +	ret = rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD,
> +					 (uintptr_t)mem, (rte_iova_t)mem, sz);
> +	if (ret) {
> +		/* Check if VFIO is not available or no device is probed */
> +		if (rte_errno == ENOTSUP || rte_errno == ENODEV) {
> +			ret = 1;
> +			goto fail;
> +		}
> +		printf("Failed to dma map whole region, ret=%d(%s)\n",
> +		       ret, rte_strerror(rte_errno));
> +		goto fail;
> +	}
> +
> +	unmap1 = (uint64_t)mem + (sz / 2);
> +	sz1 = sz / 2;
> +	unmap2 = (uint64_t)mem;
> +	sz2 = sz / 2;
> +	/* unmap the partial region */
> +	ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
> +					   unmap1, (rte_iova_t)unmap1, sz1);
> +	if (ret) {
> +		if (rte_errno == ENOTSUP) {
> +			printf("Partial dma unmap not supported\n");
> +			unmap2 = (uint64_t)mem;
> +			sz2 = sz;
> +		} else {
> +			printf("Failed to unmap second half region, ret=%d(%s)\n",
> +			       ret, rte_strerror(rte_errno));
> +			goto fail;
> +		}
> +	}
> +
> +	/* unmap the remaining region */
> +	ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
> +					   unmap2, (rte_iova_t)unmap2, sz2);
> +	if (ret) {
> +		printf("Failed to unmap remaining region, ret=%d(%s)\n", ret,
> +		       rte_strerror(rte_errno));
> +		goto fail;
> +	}
> +
> +fail:
> +	munmap(mem, sz);
> +	return ret;
> +}
> +
> +static int
> +test_vfio(void)
> +{
> +	int ret;
> +
> +	/* test for vfio dma map/unmap */
> +	ret = test_memory_vfio_dma_map();
> +	if (ret == 1) {
> +		printf("VFIO dma map/unmap unsupported\n");
> +	} else if (ret < 0) {
> +		printf("Error vfio dma map/unmap, ret=%d\n", ret);
> +		return -1;
> +	}
> +
> +	return 0;
> +}
> +
> +REGISTER_TEST_COMMAND(vfio_autotest, test_vfio);
> 

The test as written fails on a POWER9 system (see below for debug output).

IOMMU on POWER systems requires that a DMA window be defined and that 
all DMA mappings reside within that window.  In this test, the DMA 
window is defined as 0x0 to 0x4000000000, but the VA allocated in your 
test is 0x7fffb8680000, well outside that range.

I recently submitted a change in the 20.11 release which scans the 
memseg list in order to set the DMA window.  That test can be seen here:

EAL: Highest VA address in memseg list is 0x2200000000
EAL: Setting DMA window size to 0x4000000000

Can we modify the test to allocate memory out of the exsitng memseg 
allocations?

Dave

$ sudo ~/src/dpdk/build/app/test/dpdk-test --log="eal,debug" 
--iova-mode=va -l 64-127
EAL: Detected lcore 0 as core 0 on socket 0
EAL: Detected lcore 1 as core 0 on socket 0
EAL: Detected lcore 2 as core 0 on socket 0
EAL: Detected lcore 3 as core 0 on socket 0
EAL: Detected lcore 4 as core 4 on socket 0
EAL: Detected lcore 5 as core 4 on socket 0
EAL: Detected lcore 6 as core 4 on socket 0
EAL: Detected lcore 7 as core 4 on socket 0
EAL: Detected lcore 8 as core 8 on socket 0
EAL: Detected lcore 9 as core 8 on socket 0
EAL: Detected lcore 10 as core 8 on socket 0
EAL: Detected lcore 11 as core 8 on socket 0
EAL: Detected lcore 12 as core 12 on socket 0
EAL: Detected lcore 13 as core 12 on socket 0
EAL: Detected lcore 14 as core 12 on socket 0
EAL: Detected lcore 15 as core 12 on socket 0
EAL: Detected lcore 16 as core 16 on socket 0
EAL: Detected lcore 17 as core 16 on socket 0
EAL: Detected lcore 18 as core 16 on socket 0
EAL: Detected lcore 19 as core 16 on socket 0
EAL: Detected lcore 20 as core 20 on socket 0
EAL: Detected lcore 21 as core 20 on socket 0
EAL: Detected lcore 22 as core 20 on socket 0
EAL: Detected lcore 23 as core 20 on socket 0
EAL: Detected lcore 24 as core 24 on socket 0
EAL: Detected lcore 25 as core 24 on socket 0
EAL: Detected lcore 26 as core 24 on socket 0
EAL: Detected lcore 27 as core 24 on socket 0
EAL: Detected lcore 28 as core 28 on socket 0
EAL: Detected lcore 29 as core 28 on socket 0
EAL: Detected lcore 30 as core 28 on socket 0
EAL: Detected lcore 31 as core 28 on socket 0
EAL: Detected lcore 32 as core 32 on socket 0
EAL: Detected lcore 33 as core 32 on socket 0
EAL: Detected lcore 34 as core 32 on socket 0
EAL: Detected lcore 35 as core 32 on socket 0
EAL: Detected lcore 36 as core 36 on socket 0
EAL: Detected lcore 37 as core 36 on socket 0
EAL: Detected lcore 38 as core 36 on socket 0
EAL: Detected lcore 39 as core 36 on socket 0
EAL: Detected lcore 40 as core 48 on socket 0
EAL: Detected lcore 41 as core 48 on socket 0
EAL: Detected lcore 42 as core 48 on socket 0
EAL: Detected lcore 43 as core 48 on socket 0
EAL: Detected lcore 44 as core 52 on socket 0
EAL: Detected lcore 45 as core 52 on socket 0
EAL: Detected lcore 46 as core 52 on socket 0
EAL: Detected lcore 47 as core 52 on socket 0
EAL: Detected lcore 48 as core 72 on socket 0
EAL: Detected lcore 49 as core 72 on socket 0
EAL: Detected lcore 50 as core 72 on socket 0
EAL: Detected lcore 51 as core 72 on socket 0
EAL: Detected lcore 52 as core 76 on socket 0
EAL: Detected lcore 53 as core 76 on socket 0
EAL: Detected lcore 54 as core 76 on socket 0
EAL: Detected lcore 55 as core 76 on socket 0
EAL: Detected lcore 56 as core 80 on socket 0
EAL: Detected lcore 57 as core 80 on socket 0
EAL: Detected lcore 58 as core 80 on socket 0
EAL: Detected lcore 59 as core 80 on socket 0
EAL: Detected lcore 60 as core 84 on socket 0
EAL: Detected lcore 61 as core 84 on socket 0
EAL: Detected lcore 62 as core 84 on socket 0
EAL: Detected lcore 63 as core 84 on socket 0
EAL: Detected lcore 64 as core 2048 on socket 8
EAL: Detected lcore 65 as core 2048 on socket 8
EAL: Detected lcore 66 as core 2048 on socket 8
EAL: Detected lcore 67 as core 2048 on socket 8
EAL: Detected lcore 68 as core 2052 on socket 8
EAL: Detected lcore 69 as core 2052 on socket 8
EAL: Detected lcore 70 as core 2052 on socket 8
EAL: Detected lcore 71 as core 2052 on socket 8
EAL: Detected lcore 72 as core 2056 on socket 8
EAL: Detected lcore 73 as core 2056 on socket 8
EAL: Detected lcore 74 as core 2056 on socket 8
EAL: Detected lcore 75 as core 2056 on socket 8
EAL: Detected lcore 76 as core 2060 on socket 8
EAL: Detected lcore 77 as core 2060 on socket 8
EAL: Detected lcore 78 as core 2060 on socket 8
EAL: Detected lcore 79 as core 2060 on socket 8
EAL: Detected lcore 80 as core 2072 on socket 8
EAL: Detected lcore 81 as core 2072 on socket 8
EAL: Detected lcore 82 as core 2072 on socket 8
EAL: Detected lcore 83 as core 2072 on socket 8
EAL: Detected lcore 84 as core 2076 on socket 8
EAL: Detected lcore 85 as core 2076 on socket 8
EAL: Detected lcore 86 as core 2076 on socket 8
EAL: Detected lcore 87 as core 2076 on socket 8
EAL: Detected lcore 88 as core 2080 on socket 8
EAL: Detected lcore 89 as core 2080 on socket 8
EAL: Detected lcore 90 as core 2080 on socket 8
EAL: Detected lcore 91 as core 2080 on socket 8
EAL: Detected lcore 92 as core 2084 on socket 8
EAL: Detected lcore 93 as core 2084 on socket 8
EAL: Detected lcore 94 as core 2084 on socket 8
EAL: Detected lcore 95 as core 2084 on socket 8
EAL: Detected lcore 96 as core 2088 on socket 8
EAL: Detected lcore 97 as core 2088 on socket 8
EAL: Detected lcore 98 as core 2088 on socket 8
EAL: Detected lcore 99 as core 2088 on socket 8
EAL: Detected lcore 100 as core 2092 on socket 8
EAL: Detected lcore 101 as core 2092 on socket 8
EAL: Detected lcore 102 as core 2092 on socket 8
EAL: Detected lcore 103 as core 2092 on socket 8
EAL: Detected lcore 104 as core 2096 on socket 8
EAL: Detected lcore 105 as core 2096 on socket 8
EAL: Detected lcore 106 as core 2096 on socket 8
EAL: Detected lcore 107 as core 2096 on socket 8
EAL: Detected lcore 108 as core 2100 on socket 8
EAL: Detected lcore 109 as core 2100 on socket 8
EAL: Detected lcore 110 as core 2100 on socket 8
EAL: Detected lcore 111 as core 2100 on socket 8
EAL: Detected lcore 112 as core 2120 on socket 8
EAL: Detected lcore 113 as core 2120 on socket 8
EAL: Detected lcore 114 as core 2120 on socket 8
EAL: Detected lcore 115 as core 2120 on socket 8
EAL: Detected lcore 116 as core 2124 on socket 8
EAL: Detected lcore 117 as core 2124 on socket 8
EAL: Detected lcore 118 as core 2124 on socket 8
EAL: Detected lcore 119 as core 2124 on socket 8
EAL: Detected lcore 120 as core 2136 on socket 8
EAL: Detected lcore 121 as core 2136 on socket 8
EAL: Detected lcore 122 as core 2136 on socket 8
EAL: Detected lcore 123 as core 2136 on socket 8
EAL: Detected lcore 124 as core 2140 on socket 8
EAL: Detected lcore 125 as core 2140 on socket 8
EAL: Detected lcore 126 as core 2140 on socket 8
EAL: Detected lcore 127 as core 2140 on socket 8
EAL: Support maximum 1536 logical core(s) by configuration.
EAL: Detected 128 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: Ask a virtual area of 0x10000 bytes
EAL: Virtual area found at 0x100000000 (size = 0x10000)
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: DPAA Bus not present. Skipping.
EAL: VFIO PCI modules not loaded
EAL: Selected IOVA mode 'VA'
EAL: 2 hugepages of size 2097152 reserved, but no mounted hugetlbfs 
found for that size
EAL: Probing VFIO support...
EAL:   IOMMU type 1 (Type 1) is not supported
EAL:   IOMMU type 7 (sPAPR) is supported
EAL:   IOMMU type 8 (No-IOMMU) is not supported
EAL: VFIO support initialized
EAL: Ask a virtual area of 0x30000 bytes
EAL: Virtual area found at 0x100010000 (size = 0x30000)
EAL: Setting up physically contiguous memory...
EAL: Setting maximum number of open files to 32768
EAL: Detected memory type: socket_id:0 hugepage_sz:1073741824
EAL: Detected memory type: socket_id:8 hugepage_sz:1073741824
EAL: Creating 2 segment lists: n_segs:32 socket_id:0 hugepage_sz:1073741824
EAL: Ask a virtual area of 0x10000 bytes
EAL: Virtual area found at 0x100040000 (size = 0x10000)
EAL: Memseg list allocated at socket 0, page size 0x100000kB
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0x140000000 (size = 0x800000000)
EAL: VA reserved for memseg list at 0x140000000, size 800000000
EAL: Ask a virtual area of 0x10000 bytes
EAL: Virtual area found at 0x940000000 (size = 0x10000)
EAL: Memseg list allocated at socket 0, page size 0x100000kB
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0x980000000 (size = 0x800000000)
EAL: VA reserved for memseg list at 0x980000000, size 800000000
EAL: Creating 2 segment lists: n_segs:32 socket_id:8 hugepage_sz:1073741824
EAL: Ask a virtual area of 0x10000 bytes
EAL: Virtual area found at 0x1180000000 (size = 0x10000)
EAL: Memseg list allocated at socket 8, page size 0x100000kB
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0x11c0000000 (size = 0x800000000)
EAL: VA reserved for memseg list at 0x11c0000000, size 800000000
EAL: Ask a virtual area of 0x10000 bytes
EAL: Virtual area found at 0x19c0000000 (size = 0x10000)
EAL: Memseg list allocated at socket 8, page size 0x100000kB
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0x1a00000000 (size = 0x800000000)
EAL: VA reserved for memseg list at 0x1a00000000, size 800000000
EAL: TSC frequency is ~510000 KHz
EAL: Main lcore 64 is ready (tid=7fffb8018890;cpuset=[64])
EAL: lcore 65 is ready (tid=7fffb64ad090;cpuset=[65])
EAL: lcore 66 is ready (tid=7fffb5c9d090;cpuset=[66])
EAL: lcore 67 is ready (tid=7fffb548d090;cpuset=[67])
EAL: lcore 68 is ready (tid=7fffb4c7d090;cpuset=[68])
EAL: lcore 69 is ready (tid=7fffa7ffd090;cpuset=[69])
EAL: lcore 70 is ready (tid=7fffa77ed090;cpuset=[70])
EAL: lcore 71 is ready (tid=7fffa6fdd090;cpuset=[71])
EAL: lcore 72 is ready (tid=7fffa67cd090;cpuset=[72])
EAL: lcore 73 is ready (tid=7fffa5fbd090;cpuset=[73])
EAL: lcore 74 is ready (tid=7fffa57ad090;cpuset=[74])
EAL: lcore 75 is ready (tid=7fffa4f9d090;cpuset=[75])
EAL: lcore 76 is ready (tid=7fff8fffd090;cpuset=[76])
EAL: lcore 77 is ready (tid=7fff8f7ed090;cpuset=[77])
EAL: lcore 78 is ready (tid=7fff8efdd090;cpuset=[78])
EAL: lcore 79 is ready (tid=7fff8e7cd090;cpuset=[79])
EAL: lcore 80 is ready (tid=7fff8dfbd090;cpuset=[80])
EAL: lcore 81 is ready (tid=7fff8d7ad090;cpuset=[81])
EAL: lcore 82 is ready (tid=7fff8cf9d090;cpuset=[82])
EAL: lcore 83 is ready (tid=7fff6bffd090;cpuset=[83])
EAL: lcore 84 is ready (tid=7fff6b7ed090;cpuset=[84])
EAL: lcore 85 is ready (tid=7fff6afdd090;cpuset=[85])
EAL: lcore 86 is ready (tid=7fff6a7cd090;cpuset=[86])
EAL: lcore 87 is ready (tid=7fff69fbd090;cpuset=[87])
EAL: lcore 88 is ready (tid=7fff697ad090;cpuset=[88])
EAL: lcore 89 is ready (tid=7fff68f9d090;cpuset=[89])
EAL: lcore 90 is ready (tid=7fff4bffd090;cpuset=[90])
EAL: lcore 91 is ready (tid=7fff4b7ed090;cpuset=[91])
EAL: lcore 92 is ready (tid=7fff4afdd090;cpuset=[92])
EAL: lcore 93 is ready (tid=7fff4a7cd090;cpuset=[93])
EAL: lcore 94 is ready (tid=7fff49fbd090;cpuset=[94])
EAL: lcore 95 is ready (tid=7fff497ad090;cpuset=[95])
EAL: lcore 96 is ready (tid=7fff48f9d090;cpuset=[96])
EAL: lcore 97 is ready (tid=7fff2bffd090;cpuset=[97])
EAL: lcore 98 is ready (tid=7fff2b7ed090;cpuset=[98])
EAL: lcore 99 is ready (tid=7fff2afdd090;cpuset=[99])
EAL: lcore 100 is ready (tid=7fff2a7cd090;cpuset=[100])
EAL: lcore 101 is ready (tid=7fff29fbd090;cpuset=[101])
EAL: lcore 102 is ready (tid=7fff297ad090;cpuset=[102])
EAL: lcore 103 is ready (tid=7fff28f9d090;cpuset=[103])
EAL: lcore 104 is ready (tid=7fff07ffd090;cpuset=[104])
EAL: lcore 105 is ready (tid=7ffeff7ed090;cpuset=[105])
EAL: lcore 106 is ready (tid=7fff077ed090;cpuset=[106])
EAL: lcore 107 is ready (tid=7fff06fdd090;cpuset=[107])
EAL: lcore 108 is ready (tid=7fff067cd090;cpuset=[108])
EAL: lcore 109 is ready (tid=7fff05fbd090;cpuset=[109])
EAL: lcore 110 is ready (tid=7fff057ad090;cpuset=[110])
EAL: lcore 111 is ready (tid=7fff04f9d090;cpuset=[111])
EAL: lcore 112 is ready (tid=7ffeffffd090;cpuset=[112])
EAL: lcore 113 is ready (tid=7ffefefdd090;cpuset=[113])
EAL: lcore 114 is ready (tid=7ffefe7cd090;cpuset=[114])
EAL: lcore 115 is ready (tid=7ffefdfbd090;cpuset=[115])
EAL: lcore 116 is ready (tid=7ffefd7ad090;cpuset=[116])
EAL: lcore 117 is ready (tid=7ffefcf9d090;cpuset=[117])
EAL: lcore 118 is ready (tid=7ffecfffd090;cpuset=[118])
EAL: lcore 119 is ready (tid=7ffecf7ed090;cpuset=[119])
EAL: lcore 120 is ready (tid=7ffecefdd090;cpuset=[120])
EAL: lcore 121 is ready (tid=7ffece7cd090;cpuset=[121])
EAL: lcore 122 is ready (tid=7ffecdfbd090;cpuset=[122])
EAL: lcore 123 is ready (tid=7ffecd7ad090;cpuset=[123])
EAL: lcore 124 is ready (tid=7ffeccf9d090;cpuset=[124])
EAL: lcore 125 is ready (tid=7ffe9bffd090;cpuset=[125])
EAL: lcore 126 is ready (tid=7ffe9b7ed090;cpuset=[126])
EAL: lcore 127 is ready (tid=7ffe9afdd090;cpuset=[127])
EAL: Trying to obtain current memory policy.
EAL: Setting policy MPOL_PREFERRED for socket 8
EAL: Restoring previous memory policy: 0
EAL: request: mp_malloc_sync
EAL: Heap on socket 8 was expanded by 1024MB
EAL: PCI device 0000:01:00.0 on NUMA socket 0
EAL:   probe driver: 15b3:1019 mlx5_pci
EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:01:00.0 (socket 0)
EAL: Mem event callback 'MLX5_MEM_EVENT_CB:(nil)' registered
EAL: Trying to obtain current memory policy.
EAL: Setting policy MPOL_PREFERRED for socket 0
EAL: Restoring previous memory policy: 0
EAL: Calling mem event callback 'MLX5_MEM_EVENT_CB:(nil)'
EAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 1024MB
mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
mlx5_pci: Default miss action is not supported.
EAL: PCI device 0000:01:00.1 on NUMA socket 0
EAL:   probe driver: 15b3:1019 mlx5_pci
EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:01:00.1 (socket 0)
mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
mlx5_pci: Default miss action is not supported.
EAL: PCI device 0003:01:00.0 on NUMA socket 0
EAL:   probe driver: 14e4:168a net_bnx2x
EAL:   Not managed by a supported kernel driver, skipped
EAL: PCI device 0003:01:00.1 on NUMA socket 0
EAL:   probe driver: 14e4:168a net_bnx2x
EAL:   Not managed by a supported kernel driver, skipped
EAL: PCI device 0003:01:00.2 on NUMA socket 0
EAL:   probe driver: 14e4:168a net_bnx2x
EAL:   Not managed by a supported kernel driver, skipped
EAL: PCI device 0003:01:00.3 on NUMA socket 0
EAL:   probe driver: 14e4:168a net_bnx2x
EAL:   Not managed by a supported kernel driver, skipped
EAL: PCI device 0030:01:00.0 on NUMA socket 8
EAL:   probe driver: 15b3:1019 mlx5_pci
EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0030:01:00.0 (socket 8)
mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
mlx5_pci: Default miss action is not supported.
EAL: PCI device 0030:01:00.1 on NUMA socket 8
EAL:   probe driver: 15b3:1019 mlx5_pci
EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0030:01:00.1 (socket 8)
mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
mlx5_pci: Default miss action is not supported.
EAL: PCI device 0034:01:00.0 on NUMA socket 8
EAL:   probe driver: 8086:1583 net_i40e
EAL:   set IOMMU type 1 (Type 1) failed, error 19 (No such device)
EAL:   using IOMMU type 7 (sPAPR)
EAL: Highest VA address in memseg list is 0x2200000000
EAL: Setting DMA window size to 0x4000000000
EAL: Mem event callback 'vfio_mem_event_clb:(nil)' registered
EAL: Installed memory event callback for VFIO
EAL: VFIO reports MSI-X BAR as mappable
EAL:   PCI memory mapped at 0x2200000000
EAL:   PCI memory mapped at 0x2200800000
EAL: Probe PCI driver: net_i40e (8086:1583) device: 0034:01:00.0 (socket 8)
EAL: PCI device 0034:01:00.1 on NUMA socket 8
EAL:   probe driver: 8086:1583 net_i40e
EAL: VFIO reports MSI-X BAR as mappable
EAL:   PCI memory mapped at 0x2200810000
EAL:   PCI memory mapped at 0x2201010000
EAL: Probe PCI driver: net_i40e (8086:1583) device: 0034:01:00.1 (socket 8)
APP: HPET is not enabled, using TSC as default timer
RTE>>vfio_autotest
DRC: sz = 0x20000
DRC: mem = 0x0x7fffb8680000
EAL:   dma map attempt outside DMA window
EAL: Failed to map DMA
EAL: Couldn't map new region for DMA
Failed to dma map whole region, ret=-1(No such file or directory)
Error vfio dma map/unmap, ret=-1
Test Failed
RTE>>
Nithin Dabilpuram Dec. 3, 2020, 7:14 a.m. UTC | #2
On Wed, Dec 02, 2020 at 11:23:09AM -0800, David Christensen wrote:
> 
> 
> On 12/1/20 9:46 PM, Nithin Dabilpuram wrote:
> > Test case mmap's system pages and tries to performs a user
> > DMA map and unmap both partially and fully.
> > 
> > Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
> > Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> > ---
> >   app/test/meson.build |   1 +
> >   app/test/test_vfio.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++
> >   2 files changed, 104 insertions(+)
> >   create mode 100644 app/test/test_vfio.c
> > 
> > diff --git a/app/test/meson.build b/app/test/meson.build
> > index 94fd39f..d9eedb6 100644
> > --- a/app/test/meson.build
> > +++ b/app/test/meson.build
> > @@ -139,6 +139,7 @@ test_sources = files('commands.c',
> >   	'test_trace_register.c',
> >   	'test_trace_perf.c',
> >   	'test_version.c',
> > +	'test_vfio.c',
> >   	'virtual_pmd.c'
> >   )
> > 
> > diff --git a/app/test/test_vfio.c b/app/test/test_vfio.c
> > new file mode 100644
> > index 0000000..00626d4
> > --- /dev/null
> > +++ b/app/test/test_vfio.c
> > @@ -0,0 +1,103 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(C) 2020 Marvell.
> > + */
> > +
> > +#include <stdio.h>
> > +#include <stdint.h>
> > +#include <string.h>
> > +#include <sys/mman.h>
> > +#include <unistd.h>
> > +
> > +#include <rte_common.h>
> > +#include <rte_eal.h>
> > +#include <rte_eal_paging.h>
> > +#include <rte_errno.h>
> > +#include <rte_memory.h>
> > +#include <rte_vfio.h>
> > +
> > +#include "test.h"
> > +
> > +static int
> > +test_memory_vfio_dma_map(void)
> > +{
> > +	uint64_t sz1, sz2, sz = 2 * rte_mem_page_size();
> > +	uint64_t unmap1, unmap2;
> > +	uint8_t *mem;
> > +	int ret;
> > +
> > +	/* Allocate twice size of page */
> > +	mem = mmap(NULL, sz, PROT_READ | PROT_WRITE,
> > +		   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> > +	if (mem == MAP_FAILED) {
> > +		printf("Failed to allocate memory for external heap\n");
> > +		return -1;
> > +	}
> > +
> > +	/* Force page allocation */
> > +	memset(mem, 0, sz);
> > +
> > +	/* map the whole region */
> > +	ret = rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD,
> > +					 (uintptr_t)mem, (rte_iova_t)mem, sz);
> > +	if (ret) {
> > +		/* Check if VFIO is not available or no device is probed */
> > +		if (rte_errno == ENOTSUP || rte_errno == ENODEV) {
> > +			ret = 1;
> > +			goto fail;
> > +		}
> > +		printf("Failed to dma map whole region, ret=%d(%s)\n",
> > +		       ret, rte_strerror(rte_errno));
> > +		goto fail;
> > +	}
> > +
> > +	unmap1 = (uint64_t)mem + (sz / 2);
> > +	sz1 = sz / 2;
> > +	unmap2 = (uint64_t)mem;
> > +	sz2 = sz / 2;
> > +	/* unmap the partial region */
> > +	ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
> > +					   unmap1, (rte_iova_t)unmap1, sz1);
> > +	if (ret) {
> > +		if (rte_errno == ENOTSUP) {
> > +			printf("Partial dma unmap not supported\n");
> > +			unmap2 = (uint64_t)mem;
> > +			sz2 = sz;
> > +		} else {
> > +			printf("Failed to unmap second half region, ret=%d(%s)\n",
> > +			       ret, rte_strerror(rte_errno));
> > +			goto fail;
> > +		}
> > +	}
> > +
> > +	/* unmap the remaining region */
> > +	ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
> > +					   unmap2, (rte_iova_t)unmap2, sz2);
> > +	if (ret) {
> > +		printf("Failed to unmap remaining region, ret=%d(%s)\n", ret,
> > +		       rte_strerror(rte_errno));
> > +		goto fail;
> > +	}
> > +
> > +fail:
> > +	munmap(mem, sz);
> > +	return ret;
> > +}
> > +
> > +static int
> > +test_vfio(void)
> > +{
> > +	int ret;
> > +
> > +	/* test for vfio dma map/unmap */
> > +	ret = test_memory_vfio_dma_map();
> > +	if (ret == 1) {
> > +		printf("VFIO dma map/unmap unsupported\n");
> > +	} else if (ret < 0) {
> > +		printf("Error vfio dma map/unmap, ret=%d\n", ret);
> > +		return -1;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +REGISTER_TEST_COMMAND(vfio_autotest, test_vfio);
> > 
> 
> The test as written fails on a POWER9 system (see below for debug output).
> 
> IOMMU on POWER systems requires that a DMA window be defined and that all
> DMA mappings reside within that window.  In this test, the DMA window is
> defined as 0x0 to 0x4000000000, but the VA allocated in your test is
> 0x7fffb8680000, well outside that range.
> 
> I recently submitted a change in the 20.11 release which scans the memseg
> list in order to set the DMA window.  That test can be seen here:
> 
> EAL: Highest VA address in memseg list is 0x2200000000
> EAL: Setting DMA window size to 0x4000000000
> 

I missed that thread. So basically external memory with IOVA as VA mode
is not supported in POWER9 systems as it's memseg lists can be created later
after DMA window size is fixed. Correct ?

> Can we modify the test to allocate memory out of the exsitng memseg
> allocations?

Since I'm mmapin'g normal pages for this test outside EAL, I cannot use memseg list 
range as VA as it is already reserved by memseg list's.

I can see only three options left.

#1 Use initial process VA range by using heap memory instead of mmap 
which is falling below eal's base_virtaddr for both freebsd and windows and also I think 
your DMA window will include that.

#2 Use PA from real PA range or like externel_mem_autotest(). The test currently
   is acting like IOVA as VA test but this change will make it as IOVA as PA.

#3 Disable this test for VFIO SPAPR or remove it completely.

Will #1 work in your case ?
> 
> Dave
> 
> $ sudo ~/src/dpdk/build/app/test/dpdk-test --log="eal,debug" --iova-mode=va
> -l 64-127
> EAL: Detected lcore 0 as core 0 on socket 0
> EAL: Detected lcore 1 as core 0 on socket 0
> EAL: Detected lcore 2 as core 0 on socket 0
> EAL: Detected lcore 3 as core 0 on socket 0
> EAL: Detected lcore 4 as core 4 on socket 0
> EAL: Detected lcore 5 as core 4 on socket 0
> EAL: Detected lcore 6 as core 4 on socket 0
> EAL: Detected lcore 7 as core 4 on socket 0
> EAL: Detected lcore 8 as core 8 on socket 0
> EAL: Detected lcore 9 as core 8 on socket 0
> EAL: Detected lcore 10 as core 8 on socket 0
> EAL: Detected lcore 11 as core 8 on socket 0
> EAL: Detected lcore 12 as core 12 on socket 0
> EAL: Detected lcore 13 as core 12 on socket 0
> EAL: Detected lcore 14 as core 12 on socket 0
> EAL: Detected lcore 15 as core 12 on socket 0
> EAL: Detected lcore 16 as core 16 on socket 0
> EAL: Detected lcore 17 as core 16 on socket 0
> EAL: Detected lcore 18 as core 16 on socket 0
> EAL: Detected lcore 19 as core 16 on socket 0
> EAL: Detected lcore 20 as core 20 on socket 0
> EAL: Detected lcore 21 as core 20 on socket 0
> EAL: Detected lcore 22 as core 20 on socket 0
> EAL: Detected lcore 23 as core 20 on socket 0
> EAL: Detected lcore 24 as core 24 on socket 0
> EAL: Detected lcore 25 as core 24 on socket 0
> EAL: Detected lcore 26 as core 24 on socket 0
> EAL: Detected lcore 27 as core 24 on socket 0
> EAL: Detected lcore 28 as core 28 on socket 0
> EAL: Detected lcore 29 as core 28 on socket 0
> EAL: Detected lcore 30 as core 28 on socket 0
> EAL: Detected lcore 31 as core 28 on socket 0
> EAL: Detected lcore 32 as core 32 on socket 0
> EAL: Detected lcore 33 as core 32 on socket 0
> EAL: Detected lcore 34 as core 32 on socket 0
> EAL: Detected lcore 35 as core 32 on socket 0
> EAL: Detected lcore 36 as core 36 on socket 0
> EAL: Detected lcore 37 as core 36 on socket 0
> EAL: Detected lcore 38 as core 36 on socket 0
> EAL: Detected lcore 39 as core 36 on socket 0
> EAL: Detected lcore 40 as core 48 on socket 0
> EAL: Detected lcore 41 as core 48 on socket 0
> EAL: Detected lcore 42 as core 48 on socket 0
> EAL: Detected lcore 43 as core 48 on socket 0
> EAL: Detected lcore 44 as core 52 on socket 0
> EAL: Detected lcore 45 as core 52 on socket 0
> EAL: Detected lcore 46 as core 52 on socket 0
> EAL: Detected lcore 47 as core 52 on socket 0
> EAL: Detected lcore 48 as core 72 on socket 0
> EAL: Detected lcore 49 as core 72 on socket 0
> EAL: Detected lcore 50 as core 72 on socket 0
> EAL: Detected lcore 51 as core 72 on socket 0
> EAL: Detected lcore 52 as core 76 on socket 0
> EAL: Detected lcore 53 as core 76 on socket 0
> EAL: Detected lcore 54 as core 76 on socket 0
> EAL: Detected lcore 55 as core 76 on socket 0
> EAL: Detected lcore 56 as core 80 on socket 0
> EAL: Detected lcore 57 as core 80 on socket 0
> EAL: Detected lcore 58 as core 80 on socket 0
> EAL: Detected lcore 59 as core 80 on socket 0
> EAL: Detected lcore 60 as core 84 on socket 0
> EAL: Detected lcore 61 as core 84 on socket 0
> EAL: Detected lcore 62 as core 84 on socket 0
> EAL: Detected lcore 63 as core 84 on socket 0
> EAL: Detected lcore 64 as core 2048 on socket 8
> EAL: Detected lcore 65 as core 2048 on socket 8
> EAL: Detected lcore 66 as core 2048 on socket 8
> EAL: Detected lcore 67 as core 2048 on socket 8
> EAL: Detected lcore 68 as core 2052 on socket 8
> EAL: Detected lcore 69 as core 2052 on socket 8
> EAL: Detected lcore 70 as core 2052 on socket 8
> EAL: Detected lcore 71 as core 2052 on socket 8
> EAL: Detected lcore 72 as core 2056 on socket 8
> EAL: Detected lcore 73 as core 2056 on socket 8
> EAL: Detected lcore 74 as core 2056 on socket 8
> EAL: Detected lcore 75 as core 2056 on socket 8
> EAL: Detected lcore 76 as core 2060 on socket 8
> EAL: Detected lcore 77 as core 2060 on socket 8
> EAL: Detected lcore 78 as core 2060 on socket 8
> EAL: Detected lcore 79 as core 2060 on socket 8
> EAL: Detected lcore 80 as core 2072 on socket 8
> EAL: Detected lcore 81 as core 2072 on socket 8
> EAL: Detected lcore 82 as core 2072 on socket 8
> EAL: Detected lcore 83 as core 2072 on socket 8
> EAL: Detected lcore 84 as core 2076 on socket 8
> EAL: Detected lcore 85 as core 2076 on socket 8
> EAL: Detected lcore 86 as core 2076 on socket 8
> EAL: Detected lcore 87 as core 2076 on socket 8
> EAL: Detected lcore 88 as core 2080 on socket 8
> EAL: Detected lcore 89 as core 2080 on socket 8
> EAL: Detected lcore 90 as core 2080 on socket 8
> EAL: Detected lcore 91 as core 2080 on socket 8
> EAL: Detected lcore 92 as core 2084 on socket 8
> EAL: Detected lcore 93 as core 2084 on socket 8
> EAL: Detected lcore 94 as core 2084 on socket 8
> EAL: Detected lcore 95 as core 2084 on socket 8
> EAL: Detected lcore 96 as core 2088 on socket 8
> EAL: Detected lcore 97 as core 2088 on socket 8
> EAL: Detected lcore 98 as core 2088 on socket 8
> EAL: Detected lcore 99 as core 2088 on socket 8
> EAL: Detected lcore 100 as core 2092 on socket 8
> EAL: Detected lcore 101 as core 2092 on socket 8
> EAL: Detected lcore 102 as core 2092 on socket 8
> EAL: Detected lcore 103 as core 2092 on socket 8
> EAL: Detected lcore 104 as core 2096 on socket 8
> EAL: Detected lcore 105 as core 2096 on socket 8
> EAL: Detected lcore 106 as core 2096 on socket 8
> EAL: Detected lcore 107 as core 2096 on socket 8
> EAL: Detected lcore 108 as core 2100 on socket 8
> EAL: Detected lcore 109 as core 2100 on socket 8
> EAL: Detected lcore 110 as core 2100 on socket 8
> EAL: Detected lcore 111 as core 2100 on socket 8
> EAL: Detected lcore 112 as core 2120 on socket 8
> EAL: Detected lcore 113 as core 2120 on socket 8
> EAL: Detected lcore 114 as core 2120 on socket 8
> EAL: Detected lcore 115 as core 2120 on socket 8
> EAL: Detected lcore 116 as core 2124 on socket 8
> EAL: Detected lcore 117 as core 2124 on socket 8
> EAL: Detected lcore 118 as core 2124 on socket 8
> EAL: Detected lcore 119 as core 2124 on socket 8
> EAL: Detected lcore 120 as core 2136 on socket 8
> EAL: Detected lcore 121 as core 2136 on socket 8
> EAL: Detected lcore 122 as core 2136 on socket 8
> EAL: Detected lcore 123 as core 2136 on socket 8
> EAL: Detected lcore 124 as core 2140 on socket 8
> EAL: Detected lcore 125 as core 2140 on socket 8
> EAL: Detected lcore 126 as core 2140 on socket 8
> EAL: Detected lcore 127 as core 2140 on socket 8
> EAL: Support maximum 1536 logical core(s) by configuration.
> EAL: Detected 128 lcore(s)
> EAL: Detected 2 NUMA nodes
> EAL: Ask a virtual area of 0x10000 bytes
> EAL: Virtual area found at 0x100000000 (size = 0x10000)
> EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
> EAL: DPAA Bus not present. Skipping.
> EAL: VFIO PCI modules not loaded
> EAL: Selected IOVA mode 'VA'
> EAL: 2 hugepages of size 2097152 reserved, but no mounted hugetlbfs found
> for that size
> EAL: Probing VFIO support...
> EAL:   IOMMU type 1 (Type 1) is not supported
> EAL:   IOMMU type 7 (sPAPR) is supported
> EAL:   IOMMU type 8 (No-IOMMU) is not supported
> EAL: VFIO support initialized
> EAL: Ask a virtual area of 0x30000 bytes
> EAL: Virtual area found at 0x100010000 (size = 0x30000)
> EAL: Setting up physically contiguous memory...
> EAL: Setting maximum number of open files to 32768
> EAL: Detected memory type: socket_id:0 hugepage_sz:1073741824
> EAL: Detected memory type: socket_id:8 hugepage_sz:1073741824
> EAL: Creating 2 segment lists: n_segs:32 socket_id:0 hugepage_sz:1073741824
> EAL: Ask a virtual area of 0x10000 bytes
> EAL: Virtual area found at 0x100040000 (size = 0x10000)
> EAL: Memseg list allocated at socket 0, page size 0x100000kB
> EAL: Ask a virtual area of 0x800000000 bytes
> EAL: Virtual area found at 0x140000000 (size = 0x800000000)
> EAL: VA reserved for memseg list at 0x140000000, size 800000000
> EAL: Ask a virtual area of 0x10000 bytes
> EAL: Virtual area found at 0x940000000 (size = 0x10000)
> EAL: Memseg list allocated at socket 0, page size 0x100000kB
> EAL: Ask a virtual area of 0x800000000 bytes
> EAL: Virtual area found at 0x980000000 (size = 0x800000000)
> EAL: VA reserved for memseg list at 0x980000000, size 800000000
> EAL: Creating 2 segment lists: n_segs:32 socket_id:8 hugepage_sz:1073741824
> EAL: Ask a virtual area of 0x10000 bytes
> EAL: Virtual area found at 0x1180000000 (size = 0x10000)
> EAL: Memseg list allocated at socket 8, page size 0x100000kB
> EAL: Ask a virtual area of 0x800000000 bytes
> EAL: Virtual area found at 0x11c0000000 (size = 0x800000000)
> EAL: VA reserved for memseg list at 0x11c0000000, size 800000000
> EAL: Ask a virtual area of 0x10000 bytes
> EAL: Virtual area found at 0x19c0000000 (size = 0x10000)
> EAL: Memseg list allocated at socket 8, page size 0x100000kB
> EAL: Ask a virtual area of 0x800000000 bytes
> EAL: Virtual area found at 0x1a00000000 (size = 0x800000000)
> EAL: VA reserved for memseg list at 0x1a00000000, size 800000000
> EAL: TSC frequency is ~510000 KHz
> EAL: Main lcore 64 is ready (tid=7fffb8018890;cpuset=[64])
> EAL: lcore 65 is ready (tid=7fffb64ad090;cpuset=[65])
> EAL: lcore 66 is ready (tid=7fffb5c9d090;cpuset=[66])
> EAL: lcore 67 is ready (tid=7fffb548d090;cpuset=[67])
> EAL: lcore 68 is ready (tid=7fffb4c7d090;cpuset=[68])
> EAL: lcore 69 is ready (tid=7fffa7ffd090;cpuset=[69])
> EAL: lcore 70 is ready (tid=7fffa77ed090;cpuset=[70])
> EAL: lcore 71 is ready (tid=7fffa6fdd090;cpuset=[71])
> EAL: lcore 72 is ready (tid=7fffa67cd090;cpuset=[72])
> EAL: lcore 73 is ready (tid=7fffa5fbd090;cpuset=[73])
> EAL: lcore 74 is ready (tid=7fffa57ad090;cpuset=[74])
> EAL: lcore 75 is ready (tid=7fffa4f9d090;cpuset=[75])
> EAL: lcore 76 is ready (tid=7fff8fffd090;cpuset=[76])
> EAL: lcore 77 is ready (tid=7fff8f7ed090;cpuset=[77])
> EAL: lcore 78 is ready (tid=7fff8efdd090;cpuset=[78])
> EAL: lcore 79 is ready (tid=7fff8e7cd090;cpuset=[79])
> EAL: lcore 80 is ready (tid=7fff8dfbd090;cpuset=[80])
> EAL: lcore 81 is ready (tid=7fff8d7ad090;cpuset=[81])
> EAL: lcore 82 is ready (tid=7fff8cf9d090;cpuset=[82])
> EAL: lcore 83 is ready (tid=7fff6bffd090;cpuset=[83])
> EAL: lcore 84 is ready (tid=7fff6b7ed090;cpuset=[84])
> EAL: lcore 85 is ready (tid=7fff6afdd090;cpuset=[85])
> EAL: lcore 86 is ready (tid=7fff6a7cd090;cpuset=[86])
> EAL: lcore 87 is ready (tid=7fff69fbd090;cpuset=[87])
> EAL: lcore 88 is ready (tid=7fff697ad090;cpuset=[88])
> EAL: lcore 89 is ready (tid=7fff68f9d090;cpuset=[89])
> EAL: lcore 90 is ready (tid=7fff4bffd090;cpuset=[90])
> EAL: lcore 91 is ready (tid=7fff4b7ed090;cpuset=[91])
> EAL: lcore 92 is ready (tid=7fff4afdd090;cpuset=[92])
> EAL: lcore 93 is ready (tid=7fff4a7cd090;cpuset=[93])
> EAL: lcore 94 is ready (tid=7fff49fbd090;cpuset=[94])
> EAL: lcore 95 is ready (tid=7fff497ad090;cpuset=[95])
> EAL: lcore 96 is ready (tid=7fff48f9d090;cpuset=[96])
> EAL: lcore 97 is ready (tid=7fff2bffd090;cpuset=[97])
> EAL: lcore 98 is ready (tid=7fff2b7ed090;cpuset=[98])
> EAL: lcore 99 is ready (tid=7fff2afdd090;cpuset=[99])
> EAL: lcore 100 is ready (tid=7fff2a7cd090;cpuset=[100])
> EAL: lcore 101 is ready (tid=7fff29fbd090;cpuset=[101])
> EAL: lcore 102 is ready (tid=7fff297ad090;cpuset=[102])
> EAL: lcore 103 is ready (tid=7fff28f9d090;cpuset=[103])
> EAL: lcore 104 is ready (tid=7fff07ffd090;cpuset=[104])
> EAL: lcore 105 is ready (tid=7ffeff7ed090;cpuset=[105])
> EAL: lcore 106 is ready (tid=7fff077ed090;cpuset=[106])
> EAL: lcore 107 is ready (tid=7fff06fdd090;cpuset=[107])
> EAL: lcore 108 is ready (tid=7fff067cd090;cpuset=[108])
> EAL: lcore 109 is ready (tid=7fff05fbd090;cpuset=[109])
> EAL: lcore 110 is ready (tid=7fff057ad090;cpuset=[110])
> EAL: lcore 111 is ready (tid=7fff04f9d090;cpuset=[111])
> EAL: lcore 112 is ready (tid=7ffeffffd090;cpuset=[112])
> EAL: lcore 113 is ready (tid=7ffefefdd090;cpuset=[113])
> EAL: lcore 114 is ready (tid=7ffefe7cd090;cpuset=[114])
> EAL: lcore 115 is ready (tid=7ffefdfbd090;cpuset=[115])
> EAL: lcore 116 is ready (tid=7ffefd7ad090;cpuset=[116])
> EAL: lcore 117 is ready (tid=7ffefcf9d090;cpuset=[117])
> EAL: lcore 118 is ready (tid=7ffecfffd090;cpuset=[118])
> EAL: lcore 119 is ready (tid=7ffecf7ed090;cpuset=[119])
> EAL: lcore 120 is ready (tid=7ffecefdd090;cpuset=[120])
> EAL: lcore 121 is ready (tid=7ffece7cd090;cpuset=[121])
> EAL: lcore 122 is ready (tid=7ffecdfbd090;cpuset=[122])
> EAL: lcore 123 is ready (tid=7ffecd7ad090;cpuset=[123])
> EAL: lcore 124 is ready (tid=7ffeccf9d090;cpuset=[124])
> EAL: lcore 125 is ready (tid=7ffe9bffd090;cpuset=[125])
> EAL: lcore 126 is ready (tid=7ffe9b7ed090;cpuset=[126])
> EAL: lcore 127 is ready (tid=7ffe9afdd090;cpuset=[127])
> EAL: Trying to obtain current memory policy.
> EAL: Setting policy MPOL_PREFERRED for socket 8
> EAL: Restoring previous memory policy: 0
> EAL: request: mp_malloc_sync
> EAL: Heap on socket 8 was expanded by 1024MB
> EAL: PCI device 0000:01:00.0 on NUMA socket 0
> EAL:   probe driver: 15b3:1019 mlx5_pci
> EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:01:00.0 (socket 0)
> EAL: Mem event callback 'MLX5_MEM_EVENT_CB:(nil)' registered
> EAL: Trying to obtain current memory policy.
> EAL: Setting policy MPOL_PREFERRED for socket 0
> EAL: Restoring previous memory policy: 0
> EAL: Calling mem event callback 'MLX5_MEM_EVENT_CB:(nil)'
> EAL: request: mp_malloc_sync
> EAL: Heap on socket 0 was expanded by 1024MB
> mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
> mlx5_pci: Default miss action is not supported.
> EAL: PCI device 0000:01:00.1 on NUMA socket 0
> EAL:   probe driver: 15b3:1019 mlx5_pci
> EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:01:00.1 (socket 0)
> mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
> mlx5_pci: Default miss action is not supported.
> EAL: PCI device 0003:01:00.0 on NUMA socket 0
> EAL:   probe driver: 14e4:168a net_bnx2x
> EAL:   Not managed by a supported kernel driver, skipped
> EAL: PCI device 0003:01:00.1 on NUMA socket 0
> EAL:   probe driver: 14e4:168a net_bnx2x
> EAL:   Not managed by a supported kernel driver, skipped
> EAL: PCI device 0003:01:00.2 on NUMA socket 0
> EAL:   probe driver: 14e4:168a net_bnx2x
> EAL:   Not managed by a supported kernel driver, skipped
> EAL: PCI device 0003:01:00.3 on NUMA socket 0
> EAL:   probe driver: 14e4:168a net_bnx2x
> EAL:   Not managed by a supported kernel driver, skipped
> EAL: PCI device 0030:01:00.0 on NUMA socket 8
> EAL:   probe driver: 15b3:1019 mlx5_pci
> EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0030:01:00.0 (socket 8)
> mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
> mlx5_pci: Default miss action is not supported.
> EAL: PCI device 0030:01:00.1 on NUMA socket 8
> EAL:   probe driver: 15b3:1019 mlx5_pci
> EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0030:01:00.1 (socket 8)
> mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
> mlx5_pci: Default miss action is not supported.
> EAL: PCI device 0034:01:00.0 on NUMA socket 8
> EAL:   probe driver: 8086:1583 net_i40e
> EAL:   set IOMMU type 1 (Type 1) failed, error 19 (No such device)
> EAL:   using IOMMU type 7 (sPAPR)
> EAL: Highest VA address in memseg list is 0x2200000000
> EAL: Setting DMA window size to 0x4000000000
> EAL: Mem event callback 'vfio_mem_event_clb:(nil)' registered
> EAL: Installed memory event callback for VFIO
> EAL: VFIO reports MSI-X BAR as mappable
> EAL:   PCI memory mapped at 0x2200000000
> EAL:   PCI memory mapped at 0x2200800000
> EAL: Probe PCI driver: net_i40e (8086:1583) device: 0034:01:00.0 (socket 8)
> EAL: PCI device 0034:01:00.1 on NUMA socket 8
> EAL:   probe driver: 8086:1583 net_i40e
> EAL: VFIO reports MSI-X BAR as mappable
> EAL:   PCI memory mapped at 0x2200810000
> EAL:   PCI memory mapped at 0x2201010000
> EAL: Probe PCI driver: net_i40e (8086:1583) device: 0034:01:00.1 (socket 8)
> APP: HPET is not enabled, using TSC as default timer
> RTE>>vfio_autotest
> DRC: sz = 0x20000
> DRC: mem = 0x0x7fffb8680000
> EAL:   dma map attempt outside DMA window
> EAL: Failed to map DMA
> EAL: Couldn't map new region for DMA
> Failed to dma map whole region, ret=-1(No such file or directory)
> Error vfio dma map/unmap, ret=-1
> Test Failed
> RTE>>
Nithin Dabilpuram Dec. 14, 2020, 8:24 a.m. UTC | #3
Hi David,

As mentioned below in #1, I sent v5 with memory allocated from Heap which I think
falls in initially configured DMA window 0x0 .... 0x4000000000 atleast in 
Linux as DPDK memory starts after HEAP.

Let me know if it is ok for POWER9 system.

On Thu, Dec 03, 2020 at 12:44:06PM +0530, Nithin Dabilpuram wrote:
> On Wed, Dec 02, 2020 at 11:23:09AM -0800, David Christensen wrote:
> > 
> > 
> > On 12/1/20 9:46 PM, Nithin Dabilpuram wrote:
> > > Test case mmap's system pages and tries to performs a user
> > > DMA map and unmap both partially and fully.
> > > 
> > > Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
> > > Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> > > ---
> > >   app/test/meson.build |   1 +
> > >   app/test/test_vfio.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++
> > >   2 files changed, 104 insertions(+)
> > >   create mode 100644 app/test/test_vfio.c
> > > 
> > > diff --git a/app/test/meson.build b/app/test/meson.build
> > > index 94fd39f..d9eedb6 100644
> > > --- a/app/test/meson.build
> > > +++ b/app/test/meson.build
> > > @@ -139,6 +139,7 @@ test_sources = files('commands.c',
> > >   	'test_trace_register.c',
> > >   	'test_trace_perf.c',
> > >   	'test_version.c',
> > > +	'test_vfio.c',
> > >   	'virtual_pmd.c'
> > >   )
> > > 
> > > diff --git a/app/test/test_vfio.c b/app/test/test_vfio.c
> > > new file mode 100644
> > > index 0000000..00626d4
> > > --- /dev/null
> > > +++ b/app/test/test_vfio.c
> > > @@ -0,0 +1,103 @@
> > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > + * Copyright(C) 2020 Marvell.
> > > + */
> > > +
> > > +#include <stdio.h>
> > > +#include <stdint.h>
> > > +#include <string.h>
> > > +#include <sys/mman.h>
> > > +#include <unistd.h>
> > > +
> > > +#include <rte_common.h>
> > > +#include <rte_eal.h>
> > > +#include <rte_eal_paging.h>
> > > +#include <rte_errno.h>
> > > +#include <rte_memory.h>
> > > +#include <rte_vfio.h>
> > > +
> > > +#include "test.h"
> > > +
> > > +static int
> > > +test_memory_vfio_dma_map(void)
> > > +{
> > > +	uint64_t sz1, sz2, sz = 2 * rte_mem_page_size();
> > > +	uint64_t unmap1, unmap2;
> > > +	uint8_t *mem;
> > > +	int ret;
> > > +
> > > +	/* Allocate twice size of page */
> > > +	mem = mmap(NULL, sz, PROT_READ | PROT_WRITE,
> > > +		   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> > > +	if (mem == MAP_FAILED) {
> > > +		printf("Failed to allocate memory for external heap\n");
> > > +		return -1;
> > > +	}
> > > +
> > > +	/* Force page allocation */
> > > +	memset(mem, 0, sz);
> > > +
> > > +	/* map the whole region */
> > > +	ret = rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD,
> > > +					 (uintptr_t)mem, (rte_iova_t)mem, sz);
> > > +	if (ret) {
> > > +		/* Check if VFIO is not available or no device is probed */
> > > +		if (rte_errno == ENOTSUP || rte_errno == ENODEV) {
> > > +			ret = 1;
> > > +			goto fail;
> > > +		}
> > > +		printf("Failed to dma map whole region, ret=%d(%s)\n",
> > > +		       ret, rte_strerror(rte_errno));
> > > +		goto fail;
> > > +	}
> > > +
> > > +	unmap1 = (uint64_t)mem + (sz / 2);
> > > +	sz1 = sz / 2;
> > > +	unmap2 = (uint64_t)mem;
> > > +	sz2 = sz / 2;
> > > +	/* unmap the partial region */
> > > +	ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
> > > +					   unmap1, (rte_iova_t)unmap1, sz1);
> > > +	if (ret) {
> > > +		if (rte_errno == ENOTSUP) {
> > > +			printf("Partial dma unmap not supported\n");
> > > +			unmap2 = (uint64_t)mem;
> > > +			sz2 = sz;
> > > +		} else {
> > > +			printf("Failed to unmap second half region, ret=%d(%s)\n",
> > > +			       ret, rte_strerror(rte_errno));
> > > +			goto fail;
> > > +		}
> > > +	}
> > > +
> > > +	/* unmap the remaining region */
> > > +	ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
> > > +					   unmap2, (rte_iova_t)unmap2, sz2);
> > > +	if (ret) {
> > > +		printf("Failed to unmap remaining region, ret=%d(%s)\n", ret,
> > > +		       rte_strerror(rte_errno));
> > > +		goto fail;
> > > +	}
> > > +
> > > +fail:
> > > +	munmap(mem, sz);
> > > +	return ret;
> > > +}
> > > +
> > > +static int
> > > +test_vfio(void)
> > > +{
> > > +	int ret;
> > > +
> > > +	/* test for vfio dma map/unmap */
> > > +	ret = test_memory_vfio_dma_map();
> > > +	if (ret == 1) {
> > > +		printf("VFIO dma map/unmap unsupported\n");
> > > +	} else if (ret < 0) {
> > > +		printf("Error vfio dma map/unmap, ret=%d\n", ret);
> > > +		return -1;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +REGISTER_TEST_COMMAND(vfio_autotest, test_vfio);
> > > 
> > 
> > The test as written fails on a POWER9 system (see below for debug output).
> > 
> > IOMMU on POWER systems requires that a DMA window be defined and that all
> > DMA mappings reside within that window.  In this test, the DMA window is
> > defined as 0x0 to 0x4000000000, but the VA allocated in your test is
> > 0x7fffb8680000, well outside that range.
> > 
> > I recently submitted a change in the 20.11 release which scans the memseg
> > list in order to set the DMA window.  That test can be seen here:
> > 
> > EAL: Highest VA address in memseg list is 0x2200000000
> > EAL: Setting DMA window size to 0x4000000000
> > 
> 
> I missed that thread. So basically external memory with IOVA as VA mode
> is not supported in POWER9 systems as it's memseg lists can be created later
> after DMA window size is fixed. Correct ?
> 
> > Can we modify the test to allocate memory out of the exsitng memseg
> > allocations?
> 
> Since I'm mmapin'g normal pages for this test outside EAL, I cannot use memseg list 
> range as VA as it is already reserved by memseg list's.
> 
> I can see only three options left.
> 
> #1 Use initial process VA range by using heap memory instead of mmap 
> which is falling below eal's base_virtaddr for both freebsd and windows and also I think 
> your DMA window will include that.
> 
> #2 Use PA from real PA range or like externel_mem_autotest(). The test currently
>    is acting like IOVA as VA test but this change will make it as IOVA as PA.
> 
> #3 Disable this test for VFIO SPAPR or remove it completely.
> 
> Will #1 work in your case ?
> > 
> > Dave
> > 
> > $ sudo ~/src/dpdk/build/app/test/dpdk-test --log="eal,debug" --iova-mode=va
> > -l 64-127
> > EAL: Detected lcore 0 as core 0 on socket 0
> > EAL: Detected lcore 1 as core 0 on socket 0
> > EAL: Detected lcore 2 as core 0 on socket 0
> > EAL: Detected lcore 3 as core 0 on socket 0
> > EAL: Detected lcore 4 as core 4 on socket 0
> > EAL: Detected lcore 5 as core 4 on socket 0
> > EAL: Detected lcore 6 as core 4 on socket 0
> > EAL: Detected lcore 7 as core 4 on socket 0
> > EAL: Detected lcore 8 as core 8 on socket 0
> > EAL: Detected lcore 9 as core 8 on socket 0
> > EAL: Detected lcore 10 as core 8 on socket 0
> > EAL: Detected lcore 11 as core 8 on socket 0
> > EAL: Detected lcore 12 as core 12 on socket 0
> > EAL: Detected lcore 13 as core 12 on socket 0
> > EAL: Detected lcore 14 as core 12 on socket 0
> > EAL: Detected lcore 15 as core 12 on socket 0
> > EAL: Detected lcore 16 as core 16 on socket 0
> > EAL: Detected lcore 17 as core 16 on socket 0
> > EAL: Detected lcore 18 as core 16 on socket 0
> > EAL: Detected lcore 19 as core 16 on socket 0
> > EAL: Detected lcore 20 as core 20 on socket 0
> > EAL: Detected lcore 21 as core 20 on socket 0
> > EAL: Detected lcore 22 as core 20 on socket 0
> > EAL: Detected lcore 23 as core 20 on socket 0
> > EAL: Detected lcore 24 as core 24 on socket 0
> > EAL: Detected lcore 25 as core 24 on socket 0
> > EAL: Detected lcore 26 as core 24 on socket 0
> > EAL: Detected lcore 27 as core 24 on socket 0
> > EAL: Detected lcore 28 as core 28 on socket 0
> > EAL: Detected lcore 29 as core 28 on socket 0
> > EAL: Detected lcore 30 as core 28 on socket 0
> > EAL: Detected lcore 31 as core 28 on socket 0
> > EAL: Detected lcore 32 as core 32 on socket 0
> > EAL: Detected lcore 33 as core 32 on socket 0
> > EAL: Detected lcore 34 as core 32 on socket 0
> > EAL: Detected lcore 35 as core 32 on socket 0
> > EAL: Detected lcore 36 as core 36 on socket 0
> > EAL: Detected lcore 37 as core 36 on socket 0
> > EAL: Detected lcore 38 as core 36 on socket 0
> > EAL: Detected lcore 39 as core 36 on socket 0
> > EAL: Detected lcore 40 as core 48 on socket 0
> > EAL: Detected lcore 41 as core 48 on socket 0
> > EAL: Detected lcore 42 as core 48 on socket 0
> > EAL: Detected lcore 43 as core 48 on socket 0
> > EAL: Detected lcore 44 as core 52 on socket 0
> > EAL: Detected lcore 45 as core 52 on socket 0
> > EAL: Detected lcore 46 as core 52 on socket 0
> > EAL: Detected lcore 47 as core 52 on socket 0
> > EAL: Detected lcore 48 as core 72 on socket 0
> > EAL: Detected lcore 49 as core 72 on socket 0
> > EAL: Detected lcore 50 as core 72 on socket 0
> > EAL: Detected lcore 51 as core 72 on socket 0
> > EAL: Detected lcore 52 as core 76 on socket 0
> > EAL: Detected lcore 53 as core 76 on socket 0
> > EAL: Detected lcore 54 as core 76 on socket 0
> > EAL: Detected lcore 55 as core 76 on socket 0
> > EAL: Detected lcore 56 as core 80 on socket 0
> > EAL: Detected lcore 57 as core 80 on socket 0
> > EAL: Detected lcore 58 as core 80 on socket 0
> > EAL: Detected lcore 59 as core 80 on socket 0
> > EAL: Detected lcore 60 as core 84 on socket 0
> > EAL: Detected lcore 61 as core 84 on socket 0
> > EAL: Detected lcore 62 as core 84 on socket 0
> > EAL: Detected lcore 63 as core 84 on socket 0
> > EAL: Detected lcore 64 as core 2048 on socket 8
> > EAL: Detected lcore 65 as core 2048 on socket 8
> > EAL: Detected lcore 66 as core 2048 on socket 8
> > EAL: Detected lcore 67 as core 2048 on socket 8
> > EAL: Detected lcore 68 as core 2052 on socket 8
> > EAL: Detected lcore 69 as core 2052 on socket 8
> > EAL: Detected lcore 70 as core 2052 on socket 8
> > EAL: Detected lcore 71 as core 2052 on socket 8
> > EAL: Detected lcore 72 as core 2056 on socket 8
> > EAL: Detected lcore 73 as core 2056 on socket 8
> > EAL: Detected lcore 74 as core 2056 on socket 8
> > EAL: Detected lcore 75 as core 2056 on socket 8
> > EAL: Detected lcore 76 as core 2060 on socket 8
> > EAL: Detected lcore 77 as core 2060 on socket 8
> > EAL: Detected lcore 78 as core 2060 on socket 8
> > EAL: Detected lcore 79 as core 2060 on socket 8
> > EAL: Detected lcore 80 as core 2072 on socket 8
> > EAL: Detected lcore 81 as core 2072 on socket 8
> > EAL: Detected lcore 82 as core 2072 on socket 8
> > EAL: Detected lcore 83 as core 2072 on socket 8
> > EAL: Detected lcore 84 as core 2076 on socket 8
> > EAL: Detected lcore 85 as core 2076 on socket 8
> > EAL: Detected lcore 86 as core 2076 on socket 8
> > EAL: Detected lcore 87 as core 2076 on socket 8
> > EAL: Detected lcore 88 as core 2080 on socket 8
> > EAL: Detected lcore 89 as core 2080 on socket 8
> > EAL: Detected lcore 90 as core 2080 on socket 8
> > EAL: Detected lcore 91 as core 2080 on socket 8
> > EAL: Detected lcore 92 as core 2084 on socket 8
> > EAL: Detected lcore 93 as core 2084 on socket 8
> > EAL: Detected lcore 94 as core 2084 on socket 8
> > EAL: Detected lcore 95 as core 2084 on socket 8
> > EAL: Detected lcore 96 as core 2088 on socket 8
> > EAL: Detected lcore 97 as core 2088 on socket 8
> > EAL: Detected lcore 98 as core 2088 on socket 8
> > EAL: Detected lcore 99 as core 2088 on socket 8
> > EAL: Detected lcore 100 as core 2092 on socket 8
> > EAL: Detected lcore 101 as core 2092 on socket 8
> > EAL: Detected lcore 102 as core 2092 on socket 8
> > EAL: Detected lcore 103 as core 2092 on socket 8
> > EAL: Detected lcore 104 as core 2096 on socket 8
> > EAL: Detected lcore 105 as core 2096 on socket 8
> > EAL: Detected lcore 106 as core 2096 on socket 8
> > EAL: Detected lcore 107 as core 2096 on socket 8
> > EAL: Detected lcore 108 as core 2100 on socket 8
> > EAL: Detected lcore 109 as core 2100 on socket 8
> > EAL: Detected lcore 110 as core 2100 on socket 8
> > EAL: Detected lcore 111 as core 2100 on socket 8
> > EAL: Detected lcore 112 as core 2120 on socket 8
> > EAL: Detected lcore 113 as core 2120 on socket 8
> > EAL: Detected lcore 114 as core 2120 on socket 8
> > EAL: Detected lcore 115 as core 2120 on socket 8
> > EAL: Detected lcore 116 as core 2124 on socket 8
> > EAL: Detected lcore 117 as core 2124 on socket 8
> > EAL: Detected lcore 118 as core 2124 on socket 8
> > EAL: Detected lcore 119 as core 2124 on socket 8
> > EAL: Detected lcore 120 as core 2136 on socket 8
> > EAL: Detected lcore 121 as core 2136 on socket 8
> > EAL: Detected lcore 122 as core 2136 on socket 8
> > EAL: Detected lcore 123 as core 2136 on socket 8
> > EAL: Detected lcore 124 as core 2140 on socket 8
> > EAL: Detected lcore 125 as core 2140 on socket 8
> > EAL: Detected lcore 126 as core 2140 on socket 8
> > EAL: Detected lcore 127 as core 2140 on socket 8
> > EAL: Support maximum 1536 logical core(s) by configuration.
> > EAL: Detected 128 lcore(s)
> > EAL: Detected 2 NUMA nodes
> > EAL: Ask a virtual area of 0x10000 bytes
> > EAL: Virtual area found at 0x100000000 (size = 0x10000)
> > EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
> > EAL: DPAA Bus not present. Skipping.
> > EAL: VFIO PCI modules not loaded
> > EAL: Selected IOVA mode 'VA'
> > EAL: 2 hugepages of size 2097152 reserved, but no mounted hugetlbfs found
> > for that size
> > EAL: Probing VFIO support...
> > EAL:   IOMMU type 1 (Type 1) is not supported
> > EAL:   IOMMU type 7 (sPAPR) is supported
> > EAL:   IOMMU type 8 (No-IOMMU) is not supported
> > EAL: VFIO support initialized
> > EAL: Ask a virtual area of 0x30000 bytes
> > EAL: Virtual area found at 0x100010000 (size = 0x30000)
> > EAL: Setting up physically contiguous memory...
> > EAL: Setting maximum number of open files to 32768
> > EAL: Detected memory type: socket_id:0 hugepage_sz:1073741824
> > EAL: Detected memory type: socket_id:8 hugepage_sz:1073741824
> > EAL: Creating 2 segment lists: n_segs:32 socket_id:0 hugepage_sz:1073741824
> > EAL: Ask a virtual area of 0x10000 bytes
> > EAL: Virtual area found at 0x100040000 (size = 0x10000)
> > EAL: Memseg list allocated at socket 0, page size 0x100000kB
> > EAL: Ask a virtual area of 0x800000000 bytes
> > EAL: Virtual area found at 0x140000000 (size = 0x800000000)
> > EAL: VA reserved for memseg list at 0x140000000, size 800000000
> > EAL: Ask a virtual area of 0x10000 bytes
> > EAL: Virtual area found at 0x940000000 (size = 0x10000)
> > EAL: Memseg list allocated at socket 0, page size 0x100000kB
> > EAL: Ask a virtual area of 0x800000000 bytes
> > EAL: Virtual area found at 0x980000000 (size = 0x800000000)
> > EAL: VA reserved for memseg list at 0x980000000, size 800000000
> > EAL: Creating 2 segment lists: n_segs:32 socket_id:8 hugepage_sz:1073741824
> > EAL: Ask a virtual area of 0x10000 bytes
> > EAL: Virtual area found at 0x1180000000 (size = 0x10000)
> > EAL: Memseg list allocated at socket 8, page size 0x100000kB
> > EAL: Ask a virtual area of 0x800000000 bytes
> > EAL: Virtual area found at 0x11c0000000 (size = 0x800000000)
> > EAL: VA reserved for memseg list at 0x11c0000000, size 800000000
> > EAL: Ask a virtual area of 0x10000 bytes
> > EAL: Virtual area found at 0x19c0000000 (size = 0x10000)
> > EAL: Memseg list allocated at socket 8, page size 0x100000kB
> > EAL: Ask a virtual area of 0x800000000 bytes
> > EAL: Virtual area found at 0x1a00000000 (size = 0x800000000)
> > EAL: VA reserved for memseg list at 0x1a00000000, size 800000000
> > EAL: TSC frequency is ~510000 KHz
> > EAL: Main lcore 64 is ready (tid=7fffb8018890;cpuset=[64])
> > EAL: lcore 65 is ready (tid=7fffb64ad090;cpuset=[65])
> > EAL: lcore 66 is ready (tid=7fffb5c9d090;cpuset=[66])
> > EAL: lcore 67 is ready (tid=7fffb548d090;cpuset=[67])
> > EAL: lcore 68 is ready (tid=7fffb4c7d090;cpuset=[68])
> > EAL: lcore 69 is ready (tid=7fffa7ffd090;cpuset=[69])
> > EAL: lcore 70 is ready (tid=7fffa77ed090;cpuset=[70])
> > EAL: lcore 71 is ready (tid=7fffa6fdd090;cpuset=[71])
> > EAL: lcore 72 is ready (tid=7fffa67cd090;cpuset=[72])
> > EAL: lcore 73 is ready (tid=7fffa5fbd090;cpuset=[73])
> > EAL: lcore 74 is ready (tid=7fffa57ad090;cpuset=[74])
> > EAL: lcore 75 is ready (tid=7fffa4f9d090;cpuset=[75])
> > EAL: lcore 76 is ready (tid=7fff8fffd090;cpuset=[76])
> > EAL: lcore 77 is ready (tid=7fff8f7ed090;cpuset=[77])
> > EAL: lcore 78 is ready (tid=7fff8efdd090;cpuset=[78])
> > EAL: lcore 79 is ready (tid=7fff8e7cd090;cpuset=[79])
> > EAL: lcore 80 is ready (tid=7fff8dfbd090;cpuset=[80])
> > EAL: lcore 81 is ready (tid=7fff8d7ad090;cpuset=[81])
> > EAL: lcore 82 is ready (tid=7fff8cf9d090;cpuset=[82])
> > EAL: lcore 83 is ready (tid=7fff6bffd090;cpuset=[83])
> > EAL: lcore 84 is ready (tid=7fff6b7ed090;cpuset=[84])
> > EAL: lcore 85 is ready (tid=7fff6afdd090;cpuset=[85])
> > EAL: lcore 86 is ready (tid=7fff6a7cd090;cpuset=[86])
> > EAL: lcore 87 is ready (tid=7fff69fbd090;cpuset=[87])
> > EAL: lcore 88 is ready (tid=7fff697ad090;cpuset=[88])
> > EAL: lcore 89 is ready (tid=7fff68f9d090;cpuset=[89])
> > EAL: lcore 90 is ready (tid=7fff4bffd090;cpuset=[90])
> > EAL: lcore 91 is ready (tid=7fff4b7ed090;cpuset=[91])
> > EAL: lcore 92 is ready (tid=7fff4afdd090;cpuset=[92])
> > EAL: lcore 93 is ready (tid=7fff4a7cd090;cpuset=[93])
> > EAL: lcore 94 is ready (tid=7fff49fbd090;cpuset=[94])
> > EAL: lcore 95 is ready (tid=7fff497ad090;cpuset=[95])
> > EAL: lcore 96 is ready (tid=7fff48f9d090;cpuset=[96])
> > EAL: lcore 97 is ready (tid=7fff2bffd090;cpuset=[97])
> > EAL: lcore 98 is ready (tid=7fff2b7ed090;cpuset=[98])
> > EAL: lcore 99 is ready (tid=7fff2afdd090;cpuset=[99])
> > EAL: lcore 100 is ready (tid=7fff2a7cd090;cpuset=[100])
> > EAL: lcore 101 is ready (tid=7fff29fbd090;cpuset=[101])
> > EAL: lcore 102 is ready (tid=7fff297ad090;cpuset=[102])
> > EAL: lcore 103 is ready (tid=7fff28f9d090;cpuset=[103])
> > EAL: lcore 104 is ready (tid=7fff07ffd090;cpuset=[104])
> > EAL: lcore 105 is ready (tid=7ffeff7ed090;cpuset=[105])
> > EAL: lcore 106 is ready (tid=7fff077ed090;cpuset=[106])
> > EAL: lcore 107 is ready (tid=7fff06fdd090;cpuset=[107])
> > EAL: lcore 108 is ready (tid=7fff067cd090;cpuset=[108])
> > EAL: lcore 109 is ready (tid=7fff05fbd090;cpuset=[109])
> > EAL: lcore 110 is ready (tid=7fff057ad090;cpuset=[110])
> > EAL: lcore 111 is ready (tid=7fff04f9d090;cpuset=[111])
> > EAL: lcore 112 is ready (tid=7ffeffffd090;cpuset=[112])
> > EAL: lcore 113 is ready (tid=7ffefefdd090;cpuset=[113])
> > EAL: lcore 114 is ready (tid=7ffefe7cd090;cpuset=[114])
> > EAL: lcore 115 is ready (tid=7ffefdfbd090;cpuset=[115])
> > EAL: lcore 116 is ready (tid=7ffefd7ad090;cpuset=[116])
> > EAL: lcore 117 is ready (tid=7ffefcf9d090;cpuset=[117])
> > EAL: lcore 118 is ready (tid=7ffecfffd090;cpuset=[118])
> > EAL: lcore 119 is ready (tid=7ffecf7ed090;cpuset=[119])
> > EAL: lcore 120 is ready (tid=7ffecefdd090;cpuset=[120])
> > EAL: lcore 121 is ready (tid=7ffece7cd090;cpuset=[121])
> > EAL: lcore 122 is ready (tid=7ffecdfbd090;cpuset=[122])
> > EAL: lcore 123 is ready (tid=7ffecd7ad090;cpuset=[123])
> > EAL: lcore 124 is ready (tid=7ffeccf9d090;cpuset=[124])
> > EAL: lcore 125 is ready (tid=7ffe9bffd090;cpuset=[125])
> > EAL: lcore 126 is ready (tid=7ffe9b7ed090;cpuset=[126])
> > EAL: lcore 127 is ready (tid=7ffe9afdd090;cpuset=[127])
> > EAL: Trying to obtain current memory policy.
> > EAL: Setting policy MPOL_PREFERRED for socket 8
> > EAL: Restoring previous memory policy: 0
> > EAL: request: mp_malloc_sync
> > EAL: Heap on socket 8 was expanded by 1024MB
> > EAL: PCI device 0000:01:00.0 on NUMA socket 0
> > EAL:   probe driver: 15b3:1019 mlx5_pci
> > EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:01:00.0 (socket 0)
> > EAL: Mem event callback 'MLX5_MEM_EVENT_CB:(nil)' registered
> > EAL: Trying to obtain current memory policy.
> > EAL: Setting policy MPOL_PREFERRED for socket 0
> > EAL: Restoring previous memory policy: 0
> > EAL: Calling mem event callback 'MLX5_MEM_EVENT_CB:(nil)'
> > EAL: request: mp_malloc_sync
> > EAL: Heap on socket 0 was expanded by 1024MB
> > mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
> > mlx5_pci: Default miss action is not supported.
> > EAL: PCI device 0000:01:00.1 on NUMA socket 0
> > EAL:   probe driver: 15b3:1019 mlx5_pci
> > EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:01:00.1 (socket 0)
> > mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
> > mlx5_pci: Default miss action is not supported.
> > EAL: PCI device 0003:01:00.0 on NUMA socket 0
> > EAL:   probe driver: 14e4:168a net_bnx2x
> > EAL:   Not managed by a supported kernel driver, skipped
> > EAL: PCI device 0003:01:00.1 on NUMA socket 0
> > EAL:   probe driver: 14e4:168a net_bnx2x
> > EAL:   Not managed by a supported kernel driver, skipped
> > EAL: PCI device 0003:01:00.2 on NUMA socket 0
> > EAL:   probe driver: 14e4:168a net_bnx2x
> > EAL:   Not managed by a supported kernel driver, skipped
> > EAL: PCI device 0003:01:00.3 on NUMA socket 0
> > EAL:   probe driver: 14e4:168a net_bnx2x
> > EAL:   Not managed by a supported kernel driver, skipped
> > EAL: PCI device 0030:01:00.0 on NUMA socket 8
> > EAL:   probe driver: 15b3:1019 mlx5_pci
> > EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0030:01:00.0 (socket 8)
> > mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
> > mlx5_pci: Default miss action is not supported.
> > EAL: PCI device 0030:01:00.1 on NUMA socket 8
> > EAL:   probe driver: 15b3:1019 mlx5_pci
> > EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0030:01:00.1 (socket 8)
> > mlx5_pci: Size 0xFFFF is not power of 2, will be aligned to 0x10000.
> > mlx5_pci: Default miss action is not supported.
> > EAL: PCI device 0034:01:00.0 on NUMA socket 8
> > EAL:   probe driver: 8086:1583 net_i40e
> > EAL:   set IOMMU type 1 (Type 1) failed, error 19 (No such device)
> > EAL:   using IOMMU type 7 (sPAPR)
> > EAL: Highest VA address in memseg list is 0x2200000000
> > EAL: Setting DMA window size to 0x4000000000
> > EAL: Mem event callback 'vfio_mem_event_clb:(nil)' registered
> > EAL: Installed memory event callback for VFIO
> > EAL: VFIO reports MSI-X BAR as mappable
> > EAL:   PCI memory mapped at 0x2200000000
> > EAL:   PCI memory mapped at 0x2200800000
> > EAL: Probe PCI driver: net_i40e (8086:1583) device: 0034:01:00.0 (socket 8)
> > EAL: PCI device 0034:01:00.1 on NUMA socket 8
> > EAL:   probe driver: 8086:1583 net_i40e
> > EAL: VFIO reports MSI-X BAR as mappable
> > EAL:   PCI memory mapped at 0x2200810000
> > EAL:   PCI memory mapped at 0x2201010000
> > EAL: Probe PCI driver: net_i40e (8086:1583) device: 0034:01:00.1 (socket 8)
> > APP: HPET is not enabled, using TSC as default timer
> > RTE>>vfio_autotest
> > DRC: sz = 0x20000
> > DRC: mem = 0x0x7fffb8680000
> > EAL:   dma map attempt outside DMA window
> > EAL: Failed to map DMA
> > EAL: Couldn't map new region for DMA
> > Failed to dma map whole region, ret=-1(No such file or directory)
> > Error vfio dma map/unmap, ret=-1
> > Test Failed
> > RTE>>

Patch
diff mbox series

diff --git a/app/test/meson.build b/app/test/meson.build
index 94fd39f..d9eedb6 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -139,6 +139,7 @@  test_sources = files('commands.c',
 	'test_trace_register.c',
 	'test_trace_perf.c',
 	'test_version.c',
+	'test_vfio.c',
 	'virtual_pmd.c'
 )
 
diff --git a/app/test/test_vfio.c b/app/test/test_vfio.c
new file mode 100644
index 0000000..00626d4
--- /dev/null
+++ b/app/test/test_vfio.c
@@ -0,0 +1,103 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2020 Marvell.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_errno.h>
+#include <rte_memory.h>
+#include <rte_vfio.h>
+
+#include "test.h"
+
+static int
+test_memory_vfio_dma_map(void)
+{
+	uint64_t sz1, sz2, sz = 2 * rte_mem_page_size();
+	uint64_t unmap1, unmap2;
+	uint8_t *mem;
+	int ret;
+
+	/* Allocate twice size of page */
+	mem = mmap(NULL, sz, PROT_READ | PROT_WRITE,
+		   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (mem == MAP_FAILED) {
+		printf("Failed to allocate memory for external heap\n");
+		return -1;
+	}
+
+	/* Force page allocation */
+	memset(mem, 0, sz);
+
+	/* map the whole region */
+	ret = rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD,
+					 (uintptr_t)mem, (rte_iova_t)mem, sz);
+	if (ret) {
+		/* Check if VFIO is not available or no device is probed */
+		if (rte_errno == ENOTSUP || rte_errno == ENODEV) {
+			ret = 1;
+			goto fail;
+		}
+		printf("Failed to dma map whole region, ret=%d(%s)\n",
+		       ret, rte_strerror(rte_errno));
+		goto fail;
+	}
+
+	unmap1 = (uint64_t)mem + (sz / 2);
+	sz1 = sz / 2;
+	unmap2 = (uint64_t)mem;
+	sz2 = sz / 2;
+	/* unmap the partial region */
+	ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
+					   unmap1, (rte_iova_t)unmap1, sz1);
+	if (ret) {
+		if (rte_errno == ENOTSUP) {
+			printf("Partial dma unmap not supported\n");
+			unmap2 = (uint64_t)mem;
+			sz2 = sz;
+		} else {
+			printf("Failed to unmap second half region, ret=%d(%s)\n",
+			       ret, rte_strerror(rte_errno));
+			goto fail;
+		}
+	}
+
+	/* unmap the remaining region */
+	ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
+					   unmap2, (rte_iova_t)unmap2, sz2);
+	if (ret) {
+		printf("Failed to unmap remaining region, ret=%d(%s)\n", ret,
+		       rte_strerror(rte_errno));
+		goto fail;
+	}
+
+fail:
+	munmap(mem, sz);
+	return ret;
+}
+
+static int
+test_vfio(void)
+{
+	int ret;
+
+	/* test for vfio dma map/unmap */
+	ret = test_memory_vfio_dma_map();
+	if (ret == 1) {
+		printf("VFIO dma map/unmap unsupported\n");
+	} else if (ret < 0) {
+		printf("Error vfio dma map/unmap, ret=%d\n", ret);
+		return -1;
+	}
+
+	return 0;
+}
+
+REGISTER_TEST_COMMAND(vfio_autotest, test_vfio);