common/mlx5: Optimize mlx5 mempool get extmem
Checks
Commit Message
From: John Romein <romein@astron.nl>
This patch reduces the time to allocate and register tens of gigabytes
of GPU memory from hours to seconds, by sorting the heap only once
instead of for each object in the mempool.
Fixes: 690b2a8 ("common/mlx5: add mempool registration facilities")
Signed-off-by: John Romein <romein@astron.nl>
---
NOTE: this is a post of https://github.com/DPDK/dpdk/pull/70 to the
mailing list.
drivers/common/mlx5/mlx5_common_mr.c | 69 ++++++++--------------------
1 file changed, 20 insertions(+), 49 deletions(-)
---
2.40.1
@@ -1389,63 +1389,23 @@ mlx5_mempool_get_chunks(struct rte_mempool *mp, struct mlx5_range **out,
return 0;
}
-struct mlx5_mempool_get_extmem_data {
- struct mlx5_range *heap;
- unsigned int heap_size;
- int ret;
-};
-
static void
mlx5_mempool_get_extmem_cb(struct rte_mempool *mp, void *opaque,
void *obj, unsigned int obj_idx)
{
- struct mlx5_mempool_get_extmem_data *data = opaque;
+ struct mlx5_range *heap = opaque;
struct rte_mbuf *mbuf = obj;
uintptr_t addr = (uintptr_t)mbuf->buf_addr;
- struct mlx5_range *seg, *heap;
struct rte_memseg_list *msl;
size_t page_size;
uintptr_t page_start;
- unsigned int pos = 0, len = data->heap_size, delta;
RTE_SET_USED(mp);
- RTE_SET_USED(obj_idx);
- if (data->ret < 0)
- return;
- /* Binary search for an already visited page. */
- while (len > 1) {
- delta = len / 2;
- if (addr < data->heap[pos + delta].start) {
- len = delta;
- } else {
- pos += delta;
- len -= delta;
- }
- }
- if (data->heap != NULL) {
- seg = &data->heap[pos];
- if (seg->start <= addr && addr < seg->end)
- return;
- }
- /* Determine the page boundaries and remember them. */
- heap = realloc(data->heap, sizeof(heap[0]) * (data->heap_size + 1));
- if (heap == NULL) {
- free(data->heap);
- data->heap = NULL;
- data->ret = -1;
- return;
- }
- data->heap = heap;
- data->heap_size++;
- seg = &heap[data->heap_size - 1];
msl = rte_mem_virt2memseg_list((void *)addr);
page_size = msl != NULL ? msl->page_sz : rte_mem_page_size();
page_start = RTE_PTR_ALIGN_FLOOR(addr, page_size);
- seg->start = page_start;
- seg->end = page_start + page_size;
- /* Maintain the heap order. */
- qsort(data->heap, data->heap_size, sizeof(heap[0]),
- mlx5_range_compare_start);
+ heap[obj_idx].start = page_start;
+ heap[obj_idx].end = page_start + page_size;
}
/**
@@ -1457,15 +1417,26 @@ static int
mlx5_mempool_get_extmem(struct rte_mempool *mp, struct mlx5_range **out,
unsigned int *out_n)
{
- struct mlx5_mempool_get_extmem_data data;
+ unsigned out_size = 1;
+ struct mlx5_range *heap;
DRV_LOG(DEBUG, "Recovering external pinned pages of mempool %s",
mp->name);
- memset(&data, 0, sizeof(data));
- rte_mempool_obj_iter(mp, mlx5_mempool_get_extmem_cb, &data);
- *out = data.heap;
- *out_n = data.heap_size;
- return data.ret;
+ heap = malloc(mp->size * sizeof(struct mlx5_range));
+ if (heap == NULL)
+ return -1;
+ rte_mempool_obj_iter(mp, mlx5_mempool_get_extmem_cb, heap);
+ qsort(heap, mp->size, sizeof heap[0], mlx5_range_compare_start);
+ /* remove duplicates */
+ for (unsigned i = 1; i < mp->size; i ++)
+ if (heap[out_size - 1].start != heap[i].start)
+ heap[out_size ++] = heap[i];
+ heap = realloc(heap, out_size * sizeof(struct mlx5_range));
+ if (heap == NULL)
+ return -1;
+ *out = heap;
+ *out_n = out_size;
+ return 0;
}
/**