diff mbox series

[19.02,2/2] mem: use memfd for no-huge mode

Message ID 6b5e1de44f42e18a6a9326a06eb800c6bb8ddfdf.1542130721.git.anatoly.burakov@intel.com (mailing list archive)
State Superseded, archived
Delegated to: Thomas Monjalon
Headers
Series Allow using virtio without hugepages |

Checks

Context Check Description
ci/checkpatch success coding style OK
ci/Intel-compilation fail Compilation issues

Commit Message

Anatoly Burakov Nov. 13, 2018, 5:54 p.m. UTC 
  When running in no-huge mode, we anonymously allocate our memory.
While this works for regular NICs and vdev's, it's not suitable
for memory sharing scenarios such as virtio with vhost_user
backend.

To fix this, allocate no-huge memory using memfd, and register
it with memalloc just like any other memseg fd. This will enable
using rte_memseg_get_fd() API with --no-huge EAL flag.

Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
---
 lib/librte_eal/linuxapp/eal/eal_memory.c | 46 ++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

Comments

Tiwei Bie Nov. 28, 2018, 4:57 a.m. UTC | #1 
On Tue, Nov 13, 2018 at 05:54:48PM +0000, Anatoly Burakov wrote:
> When running in no-huge mode, we anonymously allocate our memory.
> While this works for regular NICs and vdev's, it's not suitable
> for memory sharing scenarios such as virtio with vhost_user
> backend.
> 
> To fix this, allocate no-huge memory using memfd, and register
> it with memalloc just like any other memseg fd. This will enable
> using rte_memseg_get_fd() API with --no-huge EAL flag.
> 
> Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
> ---
>  lib/librte_eal/linuxapp/eal/eal_memory.c | 46 ++++++++++++++++++++++--
>  1 file changed, 44 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c
> index 48b23ce19..8feac2c56 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_memory.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_memory.c
> @@ -25,6 +25,7 @@
>  #include <sys/time.h>
>  #include <signal.h>
>  #include <setjmp.h>
> +#include <linux/memfd.h>
>  #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
>  #include <numa.h>
>  #include <numaif.h>
> @@ -1345,12 +1346,15 @@ eal_legacy_hugepage_init(void)
>  	/* hugetlbfs can be disabled */
>  	if (internal_config.no_hugetlbfs) {
>  		struct rte_memseg_list *msl;
> +		int n_segs, cur_seg, fd, memfd, flags;
>  		uint64_t page_sz;
> -		int n_segs, cur_seg;
>  
>  		/* nohuge mode is legacy mode */
>  		internal_config.legacy_mem = 1;
>  
> +		/* nohuge mode is single-file segments mode */
> +		internal_config.single_file_segments = 1;
> +
>  		/* create a memseg list */
>  		msl = &mcfg->memsegs[0];
>  
> @@ -1363,8 +1367,36 @@ eal_legacy_hugepage_init(void)
>  			return -1;
>  		}
>  
> +		/* set up parameters for anonymous mmap */
> +		fd = -1;
> +		flags = MAP_PRIVATE | MAP_ANONYMOUS;
> +
> +		/* create a memfd and store it in the segment fd table */
> +		memfd = memfd_create("nohuge", 0);
> +		if (memfd < 0) {
> +			RTE_LOG(ERR, EAL, "Cannot create memfd: %s\n",
> +					strerror(errno));
> +			RTE_LOG(ERR, EAL, "Falling back to anonymous map\n");
> +		} else {
> +			/* we got an fd - now resize it */
> +			if (ftruncate(memfd, internal_config.memory) < 0) {
> +				RTE_LOG(ERR, EAL, "Cannot resize memfd: %s\n",
> +						strerror(errno));
> +				RTE_LOG(ERR, EAL, "Falling back to anonymous map\n");
> +				close(memfd);
> +			} else {
> +				/* creating memfd-backed file was successful.
> +				 * we want changes to memfd to be visible to
> +				 * other processes (such as vhost backend), so
> +				 * map it as shared memory.
> +				 */
> +				RTE_LOG(DEBUG, EAL, "Using memfd for anonymous memory\n");
> +				fd = memfd;
> +				flags = MAP_SHARED;
> +			}
> +		}
>  		addr = mmap(NULL, internal_config.memory, PROT_READ | PROT_WRITE,
> -				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> +				flags, fd, 0);
>  		if (addr == MAP_FAILED) {
>  			RTE_LOG(ERR, EAL, "%s: mmap() failed: %s\n", __func__,
>  					strerror(errno));
> @@ -1375,6 +1407,16 @@ eal_legacy_hugepage_init(void)
>  		msl->socket_id = 0;
>  		msl->len = internal_config.memory;
>  
> +		/* we're in single-file segments mode, so only the segment list
> +		 * fd needs to be set up.
> +		 */
> +		if (fd != -1) {
> +			if (eal_memalloc_set_seg_list_fd(0, fd) < 0) {
> +				RTE_LOG(ERR, EAL, "Cannot set up segment list fd\n");
> +				/* not a serious error, proceed */
> +			}
> +		}

Hi Anatoly,

Thanks for the work!

It seems the support for getting fd offset is missing in no-huge
mode. I got below error in virtio-user while trying this series
with --no-huge:

update_memory_region(): Failed to get offset, ms=0x10002e000 rte_errno=19

Thanks

> +
>  		/* populate memsegs. each memseg is one page long */
>  		for (cur_seg = 0; cur_seg < n_segs; cur_seg++) {
>  			arr = &msl->memseg_arr;
> -- 
> 2.17.1
Anatoly Burakov Nov. 28, 2018, 9:11 a.m. UTC | #2 
On 28-Nov-18 4:57 AM, Tiwei Bie wrote:
> On Tue, Nov 13, 2018 at 05:54:48PM +0000, Anatoly Burakov wrote:
>> When running in no-huge mode, we anonymously allocate our memory.
>> While this works for regular NICs and vdev's, it's not suitable
>> for memory sharing scenarios such as virtio with vhost_user
>> backend.
>>
>> To fix this, allocate no-huge memory using memfd, and register
>> it with memalloc just like any other memseg fd. This will enable
>> using rte_memseg_get_fd() API with --no-huge EAL flag.
>>
>> Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
>> ---
>>   lib/librte_eal/linuxapp/eal/eal_memory.c | 46 ++++++++++++++++++++++--
>>   1 file changed, 44 insertions(+), 2 deletions(-)
>>
>> diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c
>> index 48b23ce19..8feac2c56 100644
>> --- a/lib/librte_eal/linuxapp/eal/eal_memory.c
>> +++ b/lib/librte_eal/linuxapp/eal/eal_memory.c
>> @@ -25,6 +25,7 @@
>>   #include <sys/time.h>
>>   #include <signal.h>
>>   #include <setjmp.h>
>> +#include <linux/memfd.h>
>>   #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
>>   #include <numa.h>
>>   #include <numaif.h>
>> @@ -1345,12 +1346,15 @@ eal_legacy_hugepage_init(void)
>>   	/* hugetlbfs can be disabled */
>>   	if (internal_config.no_hugetlbfs) {
>>   		struct rte_memseg_list *msl;
>> +		int n_segs, cur_seg, fd, memfd, flags;
>>   		uint64_t page_sz;
>> -		int n_segs, cur_seg;
>>   
>>   		/* nohuge mode is legacy mode */
>>   		internal_config.legacy_mem = 1;
>>   
>> +		/* nohuge mode is single-file segments mode */
>> +		internal_config.single_file_segments = 1;
>> +
>>   		/* create a memseg list */
>>   		msl = &mcfg->memsegs[0];
>>   
>> @@ -1363,8 +1367,36 @@ eal_legacy_hugepage_init(void)
>>   			return -1;
>>   		}
>>   
>> +		/* set up parameters for anonymous mmap */
>> +		fd = -1;
>> +		flags = MAP_PRIVATE | MAP_ANONYMOUS;
>> +
>> +		/* create a memfd and store it in the segment fd table */
>> +		memfd = memfd_create("nohuge", 0);
>> +		if (memfd < 0) {
>> +			RTE_LOG(ERR, EAL, "Cannot create memfd: %s\n",
>> +					strerror(errno));
>> +			RTE_LOG(ERR, EAL, "Falling back to anonymous map\n");
>> +		} else {
>> +			/* we got an fd - now resize it */
>> +			if (ftruncate(memfd, internal_config.memory) < 0) {
>> +				RTE_LOG(ERR, EAL, "Cannot resize memfd: %s\n",
>> +						strerror(errno));
>> +				RTE_LOG(ERR, EAL, "Falling back to anonymous map\n");
>> +				close(memfd);
>> +			} else {
>> +				/* creating memfd-backed file was successful.
>> +				 * we want changes to memfd to be visible to
>> +				 * other processes (such as vhost backend), so
>> +				 * map it as shared memory.
>> +				 */
>> +				RTE_LOG(DEBUG, EAL, "Using memfd for anonymous memory\n");
>> +				fd = memfd;
>> +				flags = MAP_SHARED;
>> +			}
>> +		}
>>   		addr = mmap(NULL, internal_config.memory, PROT_READ | PROT_WRITE,
>> -				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
>> +				flags, fd, 0);
>>   		if (addr == MAP_FAILED) {
>>   			RTE_LOG(ERR, EAL, "%s: mmap() failed: %s\n", __func__,
>>   					strerror(errno));
>> @@ -1375,6 +1407,16 @@ eal_legacy_hugepage_init(void)
>>   		msl->socket_id = 0;
>>   		msl->len = internal_config.memory;
>>   
>> +		/* we're in single-file segments mode, so only the segment list
>> +		 * fd needs to be set up.
>> +		 */
>> +		if (fd != -1) {
>> +			if (eal_memalloc_set_seg_list_fd(0, fd) < 0) {
>> +				RTE_LOG(ERR, EAL, "Cannot set up segment list fd\n");
>> +				/* not a serious error, proceed */
>> +			}
>> +		}
> 
> Hi Anatoly,
> 
> Thanks for the work!
> 
> It seems the support for getting fd offset is missing in no-huge
> mode. I got below error in virtio-user while trying this series
> with --no-huge:
> 
> update_memory_region(): Failed to get offset, ms=0x10002e000 rte_errno=19

That's weird, it should have been working. I'll look into it, thanks!

> 
> Thanks
> 
>> +
>>   		/* populate memsegs. each memseg is one page long */
>>   		for (cur_seg = 0; cur_seg < n_segs; cur_seg++) {
>>   			arr = &msl->memseg_arr;
>> -- 
>> 2.17.1
>
diff mbox series

Patch

diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c
index 48b23ce19..8feac2c56 100644
--- a/lib/librte_eal/linuxapp/eal/eal_memory.c
+++ b/lib/librte_eal/linuxapp/eal/eal_memory.c
@@ -25,6 +25,7 @@ 
 #include <sys/time.h>
 #include <signal.h>
 #include <setjmp.h>
+#include <linux/memfd.h>
 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
 #include <numa.h>
 #include <numaif.h>
@@ -1345,12 +1346,15 @@  eal_legacy_hugepage_init(void)
 	/* hugetlbfs can be disabled */
 	if (internal_config.no_hugetlbfs) {
 		struct rte_memseg_list *msl;
+		int n_segs, cur_seg, fd, memfd, flags;
 		uint64_t page_sz;
-		int n_segs, cur_seg;
 
 		/* nohuge mode is legacy mode */
 		internal_config.legacy_mem = 1;
 
+		/* nohuge mode is single-file segments mode */
+		internal_config.single_file_segments = 1;
+
 		/* create a memseg list */
 		msl = &mcfg->memsegs[0];
 
@@ -1363,8 +1367,36 @@  eal_legacy_hugepage_init(void)
 			return -1;
 		}
 
+		/* set up parameters for anonymous mmap */
+		fd = -1;
+		flags = MAP_PRIVATE | MAP_ANONYMOUS;
+
+		/* create a memfd and store it in the segment fd table */
+		memfd = memfd_create("nohuge", 0);
+		if (memfd < 0) {
+			RTE_LOG(ERR, EAL, "Cannot create memfd: %s\n",
+					strerror(errno));
+			RTE_LOG(ERR, EAL, "Falling back to anonymous map\n");
+		} else {
+			/* we got an fd - now resize it */
+			if (ftruncate(memfd, internal_config.memory) < 0) {
+				RTE_LOG(ERR, EAL, "Cannot resize memfd: %s\n",
+						strerror(errno));
+				RTE_LOG(ERR, EAL, "Falling back to anonymous map\n");
+				close(memfd);
+			} else {
+				/* creating memfd-backed file was successful.
+				 * we want changes to memfd to be visible to
+				 * other processes (such as vhost backend), so
+				 * map it as shared memory.
+				 */
+				RTE_LOG(DEBUG, EAL, "Using memfd for anonymous memory\n");
+				fd = memfd;
+				flags = MAP_SHARED;
+			}
+		}
 		addr = mmap(NULL, internal_config.memory, PROT_READ | PROT_WRITE,
-				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+				flags, fd, 0);
 		if (addr == MAP_FAILED) {
 			RTE_LOG(ERR, EAL, "%s: mmap() failed: %s\n", __func__,
 					strerror(errno));
@@ -1375,6 +1407,16 @@  eal_legacy_hugepage_init(void)
 		msl->socket_id = 0;
 		msl->len = internal_config.memory;
 
+		/* we're in single-file segments mode, so only the segment list
+		 * fd needs to be set up.
+		 */
+		if (fd != -1) {
+			if (eal_memalloc_set_seg_list_fd(0, fd) < 0) {
+				RTE_LOG(ERR, EAL, "Cannot set up segment list fd\n");
+				/* not a serious error, proceed */
+			}
+		}
+
 		/* populate memsegs. each memseg is one page long */
 		for (cur_seg = 0; cur_seg < n_segs; cur_seg++) {
 			arr = &msl->memseg_arr;