[RFC,08/11] malloc: allow adding memory to named heaps
Checks
Commit Message
Add an API to add externally allocated memory to malloc heap. The
memory will be stored in memseg lists like regular DPDK memory.
Multiple segments are allowed within a heap. If IOVA table is
not provided, IOVA addresses are filled in with RTE_BAD_IOVA.
Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
---
lib/librte_eal/common/include/rte_malloc.h | 44 ++++++++++++++
lib/librte_eal/common/malloc_heap.c | 70 ++++++++++++++++++++++
lib/librte_eal/common/malloc_heap.h | 4 ++
lib/librte_eal/common/rte_malloc.c | 39 ++++++++++++
lib/librte_eal/rte_eal_version.map | 1 +
5 files changed, 158 insertions(+)
Comments
On Fri, Jul 6, 2018 at 2:17 PM, Anatoly Burakov <anatoly.burakov@intel.com>
wrote:
> Add an API to add externally allocated memory to malloc heap. The
> memory will be stored in memseg lists like regular DPDK memory.
> Multiple segments are allowed within a heap. If IOVA table is
> not provided, IOVA addresses are filled in with RTE_BAD_IOVA.
>
> Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
> ---
> lib/librte_eal/common/include/rte_malloc.h | 44 ++++++++++++++
> lib/librte_eal/common/malloc_heap.c | 70 ++++++++++++++++++++++
> lib/librte_eal/common/malloc_heap.h | 4 ++
> lib/librte_eal/common/rte_malloc.c | 39 ++++++++++++
> lib/librte_eal/rte_eal_version.map | 1 +
> 5 files changed, 158 insertions(+)
>
> diff --git a/lib/librte_eal/common/include/rte_malloc.h
> b/lib/librte_eal/common/include/rte_malloc.h
> index fa6de073a..5f933993b 100644
> --- a/lib/librte_eal/common/include/rte_malloc.h
> +++ b/lib/librte_eal/common/include/rte_malloc.h
> @@ -274,6 +274,50 @@ rte_free(void *ptr);
> int __rte_experimental
> rte_malloc_heap_create(const char *heap_name);
>
> +/**
> + * Add more memory to heap with specified name.
> + *
> + * @note Concurrently adding memory to or removing memory from different
> heaps
> + * is not safe.
> + *
> + * @note This function does not need to be called in multiple processes,
> as
> + * multiprocess synchronization will happen automatically as far as
> heap data
> + * is concerned. However, before accessing pointers to memory in this
> heap, it
> + * is responsibility of the user to ensure that the heap memory is
> accessible
> + * in all processes.
> + *
> + * @note Memory must be previously allocated for DPDK to be able to use
> it as a
> + * malloc heap. Failing to do so will result in undefined behavior, up
> to and
> + * including crashes.
> + *
> + * @note Adding memory to heap may fail in multiple processes scenario, as
> + * attaching to ``rte_fbarray`` structures may not always be successful
> in
> + * secondary processes.
> + *
> + * @param heap_name
> + * Name of the heap to create.
>
Name of the heap to add memory to.
> + * @param va_addr
> + * Start of virtual area to add to the heap.
> + * @param len
> + * Length of virtual area to add to the heap.
> + * @param iova_addrs
> + * Array of page IOVA addresses corresponding to each page in this
> memory
> + * area. Can be NULL, in which case page IOVA addresses will be set to
> + * RTE_BAD_IOVA.
> + * @param n_pages
> + * Number of elements in the iova_addrs array. Must be zero of
> ``iova_addrs``
> + * is NULL.
> + * @param page_sz
> + * Page size of the underlying memory.
> + *
> + * @return
> + * - 0 on successful creation.
> + * - -1 on error.
> + */
> +int __rte_experimental
> +rte_malloc_heap_add_memory(const char *heap_name, void *va_addr, size_t
> len,
> + rte_iova_t iova_addrs[], unsigned int n_pages, size_t
> page_sz);
> +
> /**
> * If malloc debug is enabled, check a memory block for header
> * and trailer markers to indicate that all is well with the block.
> diff --git a/lib/librte_eal/common/malloc_heap.c
> b/lib/librte_eal/common/malloc_heap.c
> index f5d103626..29446cac9 100644
> --- a/lib/librte_eal/common/malloc_heap.c
> +++ b/lib/librte_eal/common/malloc_heap.c
> @@ -892,6 +892,76 @@ malloc_heap_dump(struct malloc_heap *heap, FILE *f)
> rte_spinlock_unlock(&heap->lock);
> }
>
> +int
> +malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr,
> + rte_iova_t iova_addrs[], unsigned int n_pages, size_t
> page_sz)
> +{
> + struct rte_mem_config *mcfg = rte_eal_get_configuration()->m
> em_config;
> + char fbarray_name[RTE_FBARRAY_NAME_LEN];
> + struct rte_memseg_list *msl = NULL;
> + struct rte_fbarray *arr;
> + size_t seg_len = n_pages * page_sz;
> + unsigned int i;
> +
> + /* first, find a free memseg list */
> + for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
> + struct rte_memseg_list *tmp = &mcfg->memsegs[i];
> + if (tmp->base_va == NULL) {
> + msl = tmp;
> + break;
> + }
> + }
> + if (msl == NULL) {
> + RTE_LOG(ERR, EAL, "Couldn't find empty memseg list\n");
> + rte_errno = ENOSPC;
> + return -1;
> + }
> +
> + snprintf(fbarray_name, sizeof(fbarray_name) - 1, "%s_%p",
> + heap->name, va_addr);
> +
> + /* create the backing fbarray */
> + if (rte_fbarray_init(&msl->memseg_arr, fbarray_name, n_pages,
> + sizeof(struct rte_memseg)) < 0) {
> + RTE_LOG(ERR, EAL, "Couldn't create fbarray backing the
> memseg list\n");
> + return -1;
> + }
> + arr = &msl->memseg_arr;
> +
> + /* fbarray created, fill it up */
> + for (i = 0; i < n_pages; i++) {
> + struct rte_memseg *ms;
> +
> + rte_fbarray_set_used(arr, i);
> + ms = rte_fbarray_get(arr, i);
> + ms->addr = RTE_PTR_ADD(va_addr, n_pages * page_sz);
> + ms->iova = iova_addrs == NULL ? RTE_BAD_IOVA :
> iova_addrs[i];
> + ms->hugepage_sz = page_sz;
> + ms->len = page_sz;
> + ms->nchannel = rte_memory_get_nchannel();
> + ms->nrank = rte_memory_get_nrank();
> + ms->socket_id = -1; /* invalid socket ID */
> + }
> +
> + /* set up the memseg list */
> + msl->base_va = va_addr;
> + msl->page_sz = page_sz;
> + msl->socket_id = -1; /* invalid socket ID */
> + msl->version = 0;
> + msl->external = true;
> +
> + /* now, add newly minted memory to the malloc heap */
> + malloc_heap_add_memory(heap, msl, va_addr, seg_len);
> +
> + heap->total_size += seg_len;
> +
> + /* all done! */
> + RTE_LOG(DEBUG, EAL, "Added segment for heap %s starting at %p\n",
> + heap->name, va_addr);
> +
> + return 0;
> +}
> +
> int
> malloc_heap_create(struct malloc_heap *heap, const char *heap_name)
> {
> diff --git a/lib/librte_eal/common/malloc_heap.h
> b/lib/librte_eal/common/malloc_heap.h
> index aa819ef65..3be4656d0 100644
> --- a/lib/librte_eal/common/malloc_heap.h
> +++ b/lib/librte_eal/common/malloc_heap.h
> @@ -38,6 +38,10 @@ malloc_heap_alloc_on_heap_id(const char *type, size_t
> size,
> int
> malloc_heap_create(struct malloc_heap *heap, const char *heap_name);
>
> +int
> +malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr,
> + rte_iova_t iova_addrs[], unsigned int n_pages, size_t
> page_sz);
> +
> int
> malloc_heap_find_named_heap_idx(const char *name);
>
> diff --git a/lib/librte_eal/common/rte_malloc.c
> b/lib/librte_eal/common/rte_malloc.c
> index e000dc5b7..db0f604ad 100644
> --- a/lib/librte_eal/common/rte_malloc.c
> +++ b/lib/librte_eal/common/rte_malloc.c
> @@ -274,6 +274,45 @@ rte_malloc_virt2iova(const void *addr)
> return ms->iova + RTE_PTR_DIFF(addr, ms->addr);
> }
>
> +int
> +rte_malloc_heap_add_memory(const char *heap_name, void *va_addr, size_t
> len,
> + rte_iova_t iova_addrs[], unsigned int n_pages, size_t
> page_sz)
> +{
> + struct malloc_heap *heap = NULL;
> + unsigned int n;
> + int ret;
> +
> + /* iova_addrs is allowed to be NULL */
> + if (heap_name == NULL || va_addr == NULL ||
> + /* n_pages can be 0 if iova_addrs is NULL */
> + ((iova_addrs != NULL) == (n_pages == 0)) ||
> + page_sz == 0 || !rte_is_power_of_2(page_sz) ||
> + strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 ||
> + strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) ==
> + RTE_HEAP_NAME_MAX_LEN) {
> + rte_errno = EINVAL;
> + return -1;
> + }
> + /* find our heap */
> + heap = malloc_heap_find_named_heap(heap_name);
> + if (heap == NULL) {
> + rte_errno = EINVAL;
> + return -1;
> + }
> + n = len / page_sz;
> + if (n != n_pages && iova_addrs != NULL) {
> + rte_errno = EINVAL;
> + return -1;
> + }
> +
> + rte_spinlock_lock(&heap->lock);
> + ret = malloc_heap_add_external_memory(heap, va_addr, iova_addrs,
> n,
> + page_sz);
> + rte_spinlock_unlock(&heap->lock);
> +
> + return ret;
> +}
> +
> int
> rte_malloc_heap_create(const char *heap_name)
> {
> diff --git a/lib/librte_eal/rte_eal_version.map
> b/lib/librte_eal/rte_eal_version.map
> index f3c375156..6290cc910 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -280,6 +280,7 @@ EXPERIMENTAL {
> rte_malloc_dump_heaps;
> rte_malloc_from_heap;
> rte_malloc_get_stats_from_heap;
> + rte_malloc_heap_add_memory;
> rte_malloc_heap_create;
> rte_mem_alloc_validator_register;
> rte_mem_alloc_validator_unregister;
> --
> 2.17.1
>
@@ -274,6 +274,50 @@ rte_free(void *ptr);
int __rte_experimental
rte_malloc_heap_create(const char *heap_name);
+/**
+ * Add more memory to heap with specified name.
+ *
+ * @note Concurrently adding memory to or removing memory from different heaps
+ * is not safe.
+ *
+ * @note This function does not need to be called in multiple processes, as
+ * multiprocess synchronization will happen automatically as far as heap data
+ * is concerned. However, before accessing pointers to memory in this heap, it
+ * is responsibility of the user to ensure that the heap memory is accessible
+ * in all processes.
+ *
+ * @note Memory must be previously allocated for DPDK to be able to use it as a
+ * malloc heap. Failing to do so will result in undefined behavior, up to and
+ * including crashes.
+ *
+ * @note Adding memory to heap may fail in multiple processes scenario, as
+ * attaching to ``rte_fbarray`` structures may not always be successful in
+ * secondary processes.
+ *
+ * @param heap_name
+ * Name of the heap to create.
+ * @param va_addr
+ * Start of virtual area to add to the heap.
+ * @param len
+ * Length of virtual area to add to the heap.
+ * @param iova_addrs
+ * Array of page IOVA addresses corresponding to each page in this memory
+ * area. Can be NULL, in which case page IOVA addresses will be set to
+ * RTE_BAD_IOVA.
+ * @param n_pages
+ * Number of elements in the iova_addrs array. Must be zero of ``iova_addrs``
+ * is NULL.
+ * @param page_sz
+ * Page size of the underlying memory.
+ *
+ * @return
+ * - 0 on successful creation.
+ * - -1 on error.
+ */
+int __rte_experimental
+rte_malloc_heap_add_memory(const char *heap_name, void *va_addr, size_t len,
+ rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz);
+
/**
* If malloc debug is enabled, check a memory block for header
* and trailer markers to indicate that all is well with the block.
@@ -892,6 +892,76 @@ malloc_heap_dump(struct malloc_heap *heap, FILE *f)
rte_spinlock_unlock(&heap->lock);
}
+int
+malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr,
+ rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ char fbarray_name[RTE_FBARRAY_NAME_LEN];
+ struct rte_memseg_list *msl = NULL;
+ struct rte_fbarray *arr;
+ size_t seg_len = n_pages * page_sz;
+ unsigned int i;
+
+ /* first, find a free memseg list */
+ for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+ struct rte_memseg_list *tmp = &mcfg->memsegs[i];
+ if (tmp->base_va == NULL) {
+ msl = tmp;
+ break;
+ }
+ }
+ if (msl == NULL) {
+ RTE_LOG(ERR, EAL, "Couldn't find empty memseg list\n");
+ rte_errno = ENOSPC;
+ return -1;
+ }
+
+ snprintf(fbarray_name, sizeof(fbarray_name) - 1, "%s_%p",
+ heap->name, va_addr);
+
+ /* create the backing fbarray */
+ if (rte_fbarray_init(&msl->memseg_arr, fbarray_name, n_pages,
+ sizeof(struct rte_memseg)) < 0) {
+ RTE_LOG(ERR, EAL, "Couldn't create fbarray backing the memseg list\n");
+ return -1;
+ }
+ arr = &msl->memseg_arr;
+
+ /* fbarray created, fill it up */
+ for (i = 0; i < n_pages; i++) {
+ struct rte_memseg *ms;
+
+ rte_fbarray_set_used(arr, i);
+ ms = rte_fbarray_get(arr, i);
+ ms->addr = RTE_PTR_ADD(va_addr, n_pages * page_sz);
+ ms->iova = iova_addrs == NULL ? RTE_BAD_IOVA : iova_addrs[i];
+ ms->hugepage_sz = page_sz;
+ ms->len = page_sz;
+ ms->nchannel = rte_memory_get_nchannel();
+ ms->nrank = rte_memory_get_nrank();
+ ms->socket_id = -1; /* invalid socket ID */
+ }
+
+ /* set up the memseg list */
+ msl->base_va = va_addr;
+ msl->page_sz = page_sz;
+ msl->socket_id = -1; /* invalid socket ID */
+ msl->version = 0;
+ msl->external = true;
+
+ /* now, add newly minted memory to the malloc heap */
+ malloc_heap_add_memory(heap, msl, va_addr, seg_len);
+
+ heap->total_size += seg_len;
+
+ /* all done! */
+ RTE_LOG(DEBUG, EAL, "Added segment for heap %s starting at %p\n",
+ heap->name, va_addr);
+
+ return 0;
+}
+
int
malloc_heap_create(struct malloc_heap *heap, const char *heap_name)
{
@@ -38,6 +38,10 @@ malloc_heap_alloc_on_heap_id(const char *type, size_t size,
int
malloc_heap_create(struct malloc_heap *heap, const char *heap_name);
+int
+malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr,
+ rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz);
+
int
malloc_heap_find_named_heap_idx(const char *name);
@@ -274,6 +274,45 @@ rte_malloc_virt2iova(const void *addr)
return ms->iova + RTE_PTR_DIFF(addr, ms->addr);
}
+int
+rte_malloc_heap_add_memory(const char *heap_name, void *va_addr, size_t len,
+ rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz)
+{
+ struct malloc_heap *heap = NULL;
+ unsigned int n;
+ int ret;
+
+ /* iova_addrs is allowed to be NULL */
+ if (heap_name == NULL || va_addr == NULL ||
+ /* n_pages can be 0 if iova_addrs is NULL */
+ ((iova_addrs != NULL) == (n_pages == 0)) ||
+ page_sz == 0 || !rte_is_power_of_2(page_sz) ||
+ strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 ||
+ strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) ==
+ RTE_HEAP_NAME_MAX_LEN) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+ /* find our heap */
+ heap = malloc_heap_find_named_heap(heap_name);
+ if (heap == NULL) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+ n = len / page_sz;
+ if (n != n_pages && iova_addrs != NULL) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+
+ rte_spinlock_lock(&heap->lock);
+ ret = malloc_heap_add_external_memory(heap, va_addr, iova_addrs, n,
+ page_sz);
+ rte_spinlock_unlock(&heap->lock);
+
+ return ret;
+}
+
int
rte_malloc_heap_create(const char *heap_name)
{
@@ -280,6 +280,7 @@ EXPERIMENTAL {
rte_malloc_dump_heaps;
rte_malloc_from_heap;
rte_malloc_get_stats_from_heap;
+ rte_malloc_heap_add_memory;
rte_malloc_heap_create;
rte_mem_alloc_validator_register;
rte_mem_alloc_validator_unregister;