Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/67912/?format=api
{ "id": 67912, "url": "http://patches.dpdk.org/api/patches/67912/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20200407170058.9274-5-vuhuong@mellanox.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20200407170058.9274-5-vuhuong@mellanox.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20200407170058.9274-5-vuhuong@mellanox.com", "date": "2020-04-07T17:00:58", "name": "[v3,4/4] net/mlx5: modify net PMD to use common MR driver", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "9d116f75b94eb48909f28426fc1b22b083bad7bd", "submitter": { "id": 1660, "url": "http://patches.dpdk.org/api/people/1660/?format=api", "name": "Vu Pham", "email": "vuhuong@mellanox.com" }, "delegate": { "id": 3268, "url": "http://patches.dpdk.org/api/users/3268/?format=api", "username": "rasland", "first_name": "Raslan", "last_name": "Darawsheh", "email": "rasland@nvidia.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20200407170058.9274-5-vuhuong@mellanox.com/mbox/", "series": [ { "id": 9235, "url": "http://patches.dpdk.org/api/series/9235/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=9235", "date": "2020-04-07T17:00:55", "name": "refactor multi-process IPC and memory management codes to common driver", "version": 3, "mbox": "http://patches.dpdk.org/series/9235/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/67912/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/67912/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 13313A0577;\n\tTue, 7 Apr 2020 19:02:01 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id ECE191BFC4;\n\tTue, 7 Apr 2020 19:01:18 +0200 (CEST)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n by dpdk.org (Postfix) with ESMTP id 328E41BF68\n for <dev@dpdk.org>; Tue, 7 Apr 2020 19:01:15 +0200 (CEST)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n vuhuong@mellanox.com)\n with ESMTPS (AES256-SHA encrypted); 7 Apr 2020 20:01:12 +0300", "from mti-swat6.mti.labs.mlnx. (mti-swat6.mti.labs.mlnx\n [10.9.20.106])\n by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 037H12eN016779;\n Tue, 7 Apr 2020 20:01:11 +0300" ], "From": "Vu Pham <vuhuong@mellanox.com>", "To": "dev@dpdk.org", "Cc": "viacheslavo@mellanox.com, orika@mellanox.com, matan@mellanox.com,\n rasland@mellanox.com, Vu Pham <vuhuong@mellanox.com>", "Date": "Tue, 7 Apr 2020 10:00:58 -0700", "Message-Id": "<20200407170058.9274-5-vuhuong@mellanox.com>", "X-Mailer": "git-send-email 2.16.6", "In-Reply-To": "<20200407170058.9274-1-vuhuong@mellanox.com>", "References": "<20200402192105.11606-1-vuhuoong@mellanox.com>\n <20200407170058.9274-1-vuhuong@mellanox.com>", "Subject": "[dpdk-dev] [PATCH v3 4/4] net/mlx5: modify net PMD to use common MR\n\tdriver", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Modify mlx5 net pmd driver to use MR management APIs from common driver.\n\nSigned-off-by: Vu Pham <vuhuong@mellanox.com>\n---\n drivers/common/mlx5/Makefile | 1 +\n drivers/common/mlx5/meson.build | 1 +\n drivers/net/mlx5/mlx5.c | 4 +-\n drivers/net/mlx5/mlx5.h | 12 +-\n drivers/net/mlx5/mlx5_mp.c | 8 +-\n drivers/net/mlx5/mlx5_mr.c | 1169 ++------------------------------------\n drivers/net/mlx5/mlx5_mr.h | 87 +--\n drivers/net/mlx5/mlx5_rxtx.c | 1 +\n drivers/net/mlx5/mlx5_rxtx.h | 10 +-\n drivers/net/mlx5/mlx5_rxtx_vec.h | 2 +\n drivers/net/mlx5/mlx5_trigger.c | 1 +\n drivers/net/mlx5/mlx5_txq.c | 3 +-\n 12 files changed, 75 insertions(+), 1224 deletions(-)", "diff": "diff --git a/drivers/common/mlx5/Makefile b/drivers/common/mlx5/Makefile\nindex 2a88492731..26267c957a 100644\n--- a/drivers/common/mlx5/Makefile\n+++ b/drivers/common/mlx5/Makefile\n@@ -18,6 +18,7 @@ SRCS-y += mlx5_devx_cmds.c\n SRCS-y += mlx5_common.c\n SRCS-y += mlx5_nl.c\n SRCS-y += mlx5_common_mp.c\n+SRCS-y += mlx5_common_mr.c\n ifeq ($(CONFIG_RTE_IBVERBS_LINK_DLOPEN),y)\n INSTALL-y-lib += $(LIB_GLUE)\n endif\ndiff --git a/drivers/common/mlx5/meson.build b/drivers/common/mlx5/meson.build\nindex 83671861c9..175251b691 100644\n--- a/drivers/common/mlx5/meson.build\n+++ b/drivers/common/mlx5/meson.build\n@@ -56,6 +56,7 @@ sources = files(\n \t'mlx5_common.c',\n \t'mlx5_nl.c',\n \t'mlx5_common_mp.c',\n+\t'mlx5_common_mr.c',\n )\n if not dlopen_ibverbs\n \tsources += files('mlx5_glue.c')\ndiff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c\nindex 9eac8011f3..f45055d96f 100644\n--- a/drivers/net/mlx5/mlx5.c\n+++ b/drivers/net/mlx5/mlx5.c\n@@ -618,7 +618,7 @@ mlx5_alloc_shared_ibctx(const struct mlx5_dev_spawn_data *spawn,\n \t * At this point the device is not added to the memory\n \t * event list yet, context is just being created.\n \t */\n-\terr = mlx5_mr_btree_init(&sh->mr.cache,\n+\terr = mlx5_mr_btree_init(&sh->share_cache.cache,\n \t\t\t\t MLX5_MR_BTREE_CACHE_N * 2,\n \t\t\t\t spawn->pci_dev->device.numa_node);\n \tif (err) {\n@@ -690,7 +690,7 @@ mlx5_free_shared_ibctx(struct mlx5_ibv_shared *sh)\n \tLIST_REMOVE(sh, mem_event_cb);\n \trte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);\n \t/* Release created Memory Regions. */\n-\tmlx5_mr_release(sh);\n+\tmlx5_mr_release_cache(&sh->share_cache);\n \t/* Remove context from the global device list. */\n \tLIST_REMOVE(sh, next);\n \t/*\ndiff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h\nindex 9e15600afd..41b6e78369 100644\n--- a/drivers/net/mlx5/mlx5.h\n+++ b/drivers/net/mlx5/mlx5.h\n@@ -37,10 +37,10 @@\n #include <mlx5_prm.h>\n #include <mlx5_nl.h>\n #include <mlx5_common_mp.h>\n+#include <mlx5_common_mr.h>\n \n #include \"mlx5_defs.h\"\n #include \"mlx5_utils.h\"\n-#include \"mlx5_mr.h\"\n #include \"mlx5_autoconf.h\"\n \n /** Key string for IPC. */\n@@ -198,8 +198,6 @@ struct mlx5_verbs_alloc_ctx {\n \tconst void *obj; /* Pointer to the DPDK object. */\n };\n \n-LIST_HEAD(mlx5_mr_list, mlx5_mr);\n-\n /* Flow drop context necessary due to Verbs API. */\n struct mlx5_drop {\n \tstruct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */\n@@ -390,13 +388,7 @@ struct mlx5_ibv_shared {\n \tstruct ibv_device_attr_ex device_attr; /* Device properties. */\n \tLIST_ENTRY(mlx5_ibv_shared) mem_event_cb;\n \t/**< Called by memory event callback. */\n-\tstruct {\n-\t\tuint32_t dev_gen; /* Generation number to flush local caches. */\n-\t\trte_rwlock_t rwlock; /* MR Lock. */\n-\t\tstruct mlx5_mr_btree cache; /* Global MR cache table. */\n-\t\tstruct mlx5_mr_list mr_list; /* Registered MR list. */\n-\t\tstruct mlx5_mr_list mr_free_list; /* Freed MR list. */\n-\t} mr;\n+\tstruct mlx5_mr_share_cache share_cache;\n \t/* Shared DV/DR flow data section. */\n \tpthread_mutex_t dv_mutex; /* DV context mutex. */\n \tuint32_t dv_meta_mask; /* flow META metadata supported mask. */\ndiff --git a/drivers/net/mlx5/mlx5_mp.c b/drivers/net/mlx5/mlx5_mp.c\nindex 43684dbc3a..7ad322d474 100644\n--- a/drivers/net/mlx5/mlx5_mp.c\n+++ b/drivers/net/mlx5/mlx5_mp.c\n@@ -11,6 +11,7 @@\n #include <rte_string_fns.h>\n \n #include <mlx5_common_mp.h>\n+#include <mlx5_common_mr.h>\n \n #include \"mlx5.h\"\n #include \"mlx5_rxtx.h\"\n@@ -25,7 +26,7 @@ mlx5_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)\n \t\t(const struct mlx5_mp_param *)mp_msg->param;\n \tstruct rte_eth_dev *dev;\n \tstruct mlx5_priv *priv;\n-\tstruct mlx5_mr_cache entry;\n+\tstruct mr_cache_entry entry;\n \tuint32_t lkey;\n \tint ret;\n \n@@ -40,7 +41,10 @@ mlx5_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)\n \tswitch (param->type) {\n \tcase MLX5_MP_REQ_CREATE_MR:\n \t\tmp_init_msg(&priv->mp_id, &mp_res, param->type);\n-\t\tlkey = mlx5_mr_create_primary(dev, &entry, param->args.addr);\n+\t\tlkey = mlx5_mr_create_primary(priv->sh->pd,\n+\t\t\t\t\t &priv->sh->share_cache,\n+\t\t\t\t\t &entry, param->args.addr,\n+\t\t\t\t\t priv->config.mr_ext_memseg_en);\n \t\tif (lkey == UINT32_MAX)\n \t\t\tres->result = -rte_errno;\n \t\tret = rte_mp_reply(&mp_res, peer);\ndiff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c\nindex 9151992a72..2b4b3e2891 100644\n--- a/drivers/net/mlx5/mlx5_mr.c\n+++ b/drivers/net/mlx5/mlx5_mr.c\n@@ -18,6 +18,8 @@\n #include <rte_bus_pci.h>\n \n #include <mlx5_glue.h>\n+#include <mlx5_common_mp.h>\n+#include <mlx5_common_mr.h>\n \n #include \"mlx5.h\"\n #include \"mlx5_mr.h\"\n@@ -36,834 +38,6 @@ struct mr_update_mp_data {\n \tint ret;\n };\n \n-/**\n- * Expand B-tree table to a given size. Can't be called with holding\n- * memory_hotplug_lock or sh->mr.rwlock due to rte_realloc().\n- *\n- * @param bt\n- * Pointer to B-tree structure.\n- * @param n\n- * Number of entries for expansion.\n- *\n- * @return\n- * 0 on success, -1 on failure.\n- */\n-static int\n-mr_btree_expand(struct mlx5_mr_btree *bt, int n)\n-{\n-\tvoid *mem;\n-\tint ret = 0;\n-\n-\tif (n <= bt->size)\n-\t\treturn ret;\n-\t/*\n-\t * Downside of directly using rte_realloc() is that SOCKET_ID_ANY is\n-\t * used inside if there's no room to expand. Because this is a quite\n-\t * rare case and a part of very slow path, it is very acceptable.\n-\t * Initially cache_bh[] will be given practically enough space and once\n-\t * it is expanded, expansion wouldn't be needed again ever.\n-\t */\n-\tmem = rte_realloc(bt->table, n * sizeof(struct mlx5_mr_cache), 0);\n-\tif (mem == NULL) {\n-\t\t/* Not an error, B-tree search will be skipped. */\n-\t\tDRV_LOG(WARNING, \"failed to expand MR B-tree (%p) table\",\n-\t\t\t(void *)bt);\n-\t\tret = -1;\n-\t} else {\n-\t\tDRV_LOG(DEBUG, \"expanded MR B-tree table (size=%u)\", n);\n-\t\tbt->table = mem;\n-\t\tbt->size = n;\n-\t}\n-\treturn ret;\n-}\n-\n-/**\n- * Look up LKey from given B-tree lookup table, store the last index and return\n- * searched LKey.\n- *\n- * @param bt\n- * Pointer to B-tree structure.\n- * @param[out] idx\n- * Pointer to index. Even on search failure, returns index where it stops\n- * searching so that index can be used when inserting a new entry.\n- * @param addr\n- * Search key.\n- *\n- * @return\n- * Searched LKey on success, UINT32_MAX on no match.\n- */\n-static uint32_t\n-mr_btree_lookup(struct mlx5_mr_btree *bt, uint16_t *idx, uintptr_t addr)\n-{\n-\tstruct mlx5_mr_cache *lkp_tbl;\n-\tuint16_t n;\n-\tuint16_t base = 0;\n-\n-\tMLX5_ASSERT(bt != NULL);\n-\tlkp_tbl = *bt->table;\n-\tn = bt->len;\n-\t/* First entry must be NULL for comparison. */\n-\tMLX5_ASSERT(bt->len > 0 || (lkp_tbl[0].start == 0 &&\n-\t\t\t\t lkp_tbl[0].lkey == UINT32_MAX));\n-\t/* Binary search. */\n-\tdo {\n-\t\tregister uint16_t delta = n >> 1;\n-\n-\t\tif (addr < lkp_tbl[base + delta].start) {\n-\t\t\tn = delta;\n-\t\t} else {\n-\t\t\tbase += delta;\n-\t\t\tn -= delta;\n-\t\t}\n-\t} while (n > 1);\n-\tMLX5_ASSERT(addr >= lkp_tbl[base].start);\n-\t*idx = base;\n-\tif (addr < lkp_tbl[base].end)\n-\t\treturn lkp_tbl[base].lkey;\n-\t/* Not found. */\n-\treturn UINT32_MAX;\n-}\n-\n-/**\n- * Insert an entry to B-tree lookup table.\n- *\n- * @param bt\n- * Pointer to B-tree structure.\n- * @param entry\n- * Pointer to new entry to insert.\n- *\n- * @return\n- * 0 on success, -1 on failure.\n- */\n-static int\n-mr_btree_insert(struct mlx5_mr_btree *bt, struct mlx5_mr_cache *entry)\n-{\n-\tstruct mlx5_mr_cache *lkp_tbl;\n-\tuint16_t idx = 0;\n-\tsize_t shift;\n-\n-\tMLX5_ASSERT(bt != NULL);\n-\tMLX5_ASSERT(bt->len <= bt->size);\n-\tMLX5_ASSERT(bt->len > 0);\n-\tlkp_tbl = *bt->table;\n-\t/* Find out the slot for insertion. */\n-\tif (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {\n-\t\tDRV_LOG(DEBUG,\n-\t\t\t\"abort insertion to B-tree(%p): already exist at\"\n-\t\t\t\" idx=%u [0x%\" PRIxPTR \", 0x%\" PRIxPTR \") lkey=0x%x\",\n-\t\t\t(void *)bt, idx, entry->start, entry->end, entry->lkey);\n-\t\t/* Already exist, return. */\n-\t\treturn 0;\n-\t}\n-\t/* If table is full, return error. */\n-\tif (unlikely(bt->len == bt->size)) {\n-\t\tbt->overflow = 1;\n-\t\treturn -1;\n-\t}\n-\t/* Insert entry. */\n-\t++idx;\n-\tshift = (bt->len - idx) * sizeof(struct mlx5_mr_cache);\n-\tif (shift)\n-\t\tmemmove(&lkp_tbl[idx + 1], &lkp_tbl[idx], shift);\n-\tlkp_tbl[idx] = *entry;\n-\tbt->len++;\n-\tDRV_LOG(DEBUG,\n-\t\t\"inserted B-tree(%p)[%u],\"\n-\t\t\" [0x%\" PRIxPTR \", 0x%\" PRIxPTR \") lkey=0x%x\",\n-\t\t(void *)bt, idx, entry->start, entry->end, entry->lkey);\n-\treturn 0;\n-}\n-\n-/**\n- * Initialize B-tree and allocate memory for lookup table.\n- *\n- * @param bt\n- * Pointer to B-tree structure.\n- * @param n\n- * Number of entries to allocate.\n- * @param socket\n- * NUMA socket on which memory must be allocated.\n- *\n- * @return\n- * 0 on success, a negative errno value otherwise and rte_errno is set.\n- */\n-int\n-mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket)\n-{\n-\tif (bt == NULL) {\n-\t\trte_errno = EINVAL;\n-\t\treturn -rte_errno;\n-\t}\n-\tMLX5_ASSERT(!bt->table && !bt->size);\n-\tmemset(bt, 0, sizeof(*bt));\n-\tbt->table = rte_calloc_socket(\"B-tree table\",\n-\t\t\t\t n, sizeof(struct mlx5_mr_cache),\n-\t\t\t\t 0, socket);\n-\tif (bt->table == NULL) {\n-\t\trte_errno = ENOMEM;\n-\t\tDEBUG(\"failed to allocate memory for btree cache on socket %d\",\n-\t\t socket);\n-\t\treturn -rte_errno;\n-\t}\n-\tbt->size = n;\n-\t/* First entry must be NULL for binary search. */\n-\t(*bt->table)[bt->len++] = (struct mlx5_mr_cache) {\n-\t\t.lkey = UINT32_MAX,\n-\t};\n-\tDEBUG(\"initialized B-tree %p with table %p\",\n-\t (void *)bt, (void *)bt->table);\n-\treturn 0;\n-}\n-\n-/**\n- * Free B-tree resources.\n- *\n- * @param bt\n- * Pointer to B-tree structure.\n- */\n-void\n-mlx5_mr_btree_free(struct mlx5_mr_btree *bt)\n-{\n-\tif (bt == NULL)\n-\t\treturn;\n-\tDEBUG(\"freeing B-tree %p with table %p\",\n-\t (void *)bt, (void *)bt->table);\n-\trte_free(bt->table);\n-\tmemset(bt, 0, sizeof(*bt));\n-}\n-\n-/**\n- * Dump all the entries in a B-tree\n- *\n- * @param bt\n- * Pointer to B-tree structure.\n- */\n-void\n-mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused)\n-{\n-#ifdef RTE_LIBRTE_MLX5_DEBUG\n-\tint idx;\n-\tstruct mlx5_mr_cache *lkp_tbl;\n-\n-\tif (bt == NULL)\n-\t\treturn;\n-\tlkp_tbl = *bt->table;\n-\tfor (idx = 0; idx < bt->len; ++idx) {\n-\t\tstruct mlx5_mr_cache *entry = &lkp_tbl[idx];\n-\n-\t\tDEBUG(\"B-tree(%p)[%u],\"\n-\t\t \" [0x%\" PRIxPTR \", 0x%\" PRIxPTR \") lkey=0x%x\",\n-\t\t (void *)bt, idx, entry->start, entry->end, entry->lkey);\n-\t}\n-#endif\n-}\n-\n-/**\n- * Find virtually contiguous memory chunk in a given MR.\n- *\n- * @param dev\n- * Pointer to MR structure.\n- * @param[out] entry\n- * Pointer to returning MR cache entry. If not found, this will not be\n- * updated.\n- * @param start_idx\n- * Start index of the memseg bitmap.\n- *\n- * @return\n- * Next index to go on lookup.\n- */\n-static int\n-mr_find_next_chunk(struct mlx5_mr *mr, struct mlx5_mr_cache *entry,\n-\t\t int base_idx)\n-{\n-\tuintptr_t start = 0;\n-\tuintptr_t end = 0;\n-\tuint32_t idx = 0;\n-\n-\t/* MR for external memory doesn't have memseg list. */\n-\tif (mr->msl == NULL) {\n-\t\tstruct ibv_mr *ibv_mr = mr->ibv_mr;\n-\n-\t\tMLX5_ASSERT(mr->ms_bmp_n == 1);\n-\t\tMLX5_ASSERT(mr->ms_n == 1);\n-\t\tMLX5_ASSERT(base_idx == 0);\n-\t\t/*\n-\t\t * Can't search it from memseg list but get it directly from\n-\t\t * verbs MR as there's only one chunk.\n-\t\t */\n-\t\tentry->start = (uintptr_t)ibv_mr->addr;\n-\t\tentry->end = (uintptr_t)ibv_mr->addr + mr->ibv_mr->length;\n-\t\tentry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);\n-\t\t/* Returning 1 ends iteration. */\n-\t\treturn 1;\n-\t}\n-\tfor (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {\n-\t\tif (rte_bitmap_get(mr->ms_bmp, idx)) {\n-\t\t\tconst struct rte_memseg_list *msl;\n-\t\t\tconst struct rte_memseg *ms;\n-\n-\t\t\tmsl = mr->msl;\n-\t\t\tms = rte_fbarray_get(&msl->memseg_arr,\n-\t\t\t\t\t mr->ms_base_idx + idx);\n-\t\t\tMLX5_ASSERT(msl->page_sz == ms->hugepage_sz);\n-\t\t\tif (!start)\n-\t\t\t\tstart = ms->addr_64;\n-\t\t\tend = ms->addr_64 + ms->hugepage_sz;\n-\t\t} else if (start) {\n-\t\t\t/* Passed the end of a fragment. */\n-\t\t\tbreak;\n-\t\t}\n-\t}\n-\tif (start) {\n-\t\t/* Found one chunk. */\n-\t\tentry->start = start;\n-\t\tentry->end = end;\n-\t\tentry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);\n-\t}\n-\treturn idx;\n-}\n-\n-/**\n- * Insert a MR to the global B-tree cache. It may fail due to low-on-memory.\n- * Then, this entry will have to be searched by mr_lookup_dev_list() in\n- * mlx5_mr_create() on miss.\n- *\n- * @param dev\n- * Pointer to Ethernet device shared context.\n- * @param mr\n- * Pointer to MR to insert.\n- *\n- * @return\n- * 0 on success, -1 on failure.\n- */\n-static int\n-mr_insert_dev_cache(struct mlx5_ibv_shared *sh, struct mlx5_mr *mr)\n-{\n-\tunsigned int n;\n-\n-\tDRV_LOG(DEBUG, \"device %s inserting MR(%p) to global cache\",\n-\t\tsh->ibdev_name, (void *)mr);\n-\tfor (n = 0; n < mr->ms_bmp_n; ) {\n-\t\tstruct mlx5_mr_cache entry;\n-\n-\t\tmemset(&entry, 0, sizeof(entry));\n-\t\t/* Find a contiguous chunk and advance the index. */\n-\t\tn = mr_find_next_chunk(mr, &entry, n);\n-\t\tif (!entry.end)\n-\t\t\tbreak;\n-\t\tif (mr_btree_insert(&sh->mr.cache, &entry) < 0) {\n-\t\t\t/*\n-\t\t\t * Overflowed, but the global table cannot be expanded\n-\t\t\t * because of deadlock.\n-\t\t\t */\n-\t\t\treturn -1;\n-\t\t}\n-\t}\n-\treturn 0;\n-}\n-\n-/**\n- * Look up address in the original global MR list.\n- *\n- * @param sh\n- * Pointer to Ethernet device shared context.\n- * @param[out] entry\n- * Pointer to returning MR cache entry. If no match, this will not be updated.\n- * @param addr\n- * Search key.\n- *\n- * @return\n- * Found MR on match, NULL otherwise.\n- */\n-static struct mlx5_mr *\n-mr_lookup_dev_list(struct mlx5_ibv_shared *sh, struct mlx5_mr_cache *entry,\n-\t\t uintptr_t addr)\n-{\n-\tstruct mlx5_mr *mr;\n-\n-\t/* Iterate all the existing MRs. */\n-\tLIST_FOREACH(mr, &sh->mr.mr_list, mr) {\n-\t\tunsigned int n;\n-\n-\t\tif (mr->ms_n == 0)\n-\t\t\tcontinue;\n-\t\tfor (n = 0; n < mr->ms_bmp_n; ) {\n-\t\t\tstruct mlx5_mr_cache ret;\n-\n-\t\t\tmemset(&ret, 0, sizeof(ret));\n-\t\t\tn = mr_find_next_chunk(mr, &ret, n);\n-\t\t\tif (addr >= ret.start && addr < ret.end) {\n-\t\t\t\t/* Found. */\n-\t\t\t\t*entry = ret;\n-\t\t\t\treturn mr;\n-\t\t\t}\n-\t\t}\n-\t}\n-\treturn NULL;\n-}\n-\n-/**\n- * Look up address on device.\n- *\n- * @param dev\n- * Pointer to Ethernet device shared context.\n- * @param[out] entry\n- * Pointer to returning MR cache entry. If no match, this will not be updated.\n- * @param addr\n- * Search key.\n- *\n- * @return\n- * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.\n- */\n-static uint32_t\n-mr_lookup_dev(struct mlx5_ibv_shared *sh, struct mlx5_mr_cache *entry,\n-\t uintptr_t addr)\n-{\n-\tuint16_t idx;\n-\tuint32_t lkey = UINT32_MAX;\n-\tstruct mlx5_mr *mr;\n-\n-\t/*\n-\t * If the global cache has overflowed since it failed to expand the\n-\t * B-tree table, it can't have all the existing MRs. Then, the address\n-\t * has to be searched by traversing the original MR list instead, which\n-\t * is very slow path. Otherwise, the global cache is all inclusive.\n-\t */\n-\tif (!unlikely(sh->mr.cache.overflow)) {\n-\t\tlkey = mr_btree_lookup(&sh->mr.cache, &idx, addr);\n-\t\tif (lkey != UINT32_MAX)\n-\t\t\t*entry = (*sh->mr.cache.table)[idx];\n-\t} else {\n-\t\t/* Falling back to the slowest path. */\n-\t\tmr = mr_lookup_dev_list(sh, entry, addr);\n-\t\tif (mr != NULL)\n-\t\t\tlkey = entry->lkey;\n-\t}\n-\tMLX5_ASSERT(lkey == UINT32_MAX || (addr >= entry->start &&\n-\t\t\t\t\t addr < entry->end));\n-\treturn lkey;\n-}\n-\n-/**\n- * Free MR resources. MR lock must not be held to avoid a deadlock. rte_free()\n- * can raise memory free event and the callback function will spin on the lock.\n- *\n- * @param mr\n- * Pointer to MR to free.\n- */\n-static void\n-mr_free(struct mlx5_mr *mr)\n-{\n-\tif (mr == NULL)\n-\t\treturn;\n-\tDRV_LOG(DEBUG, \"freeing MR(%p):\", (void *)mr);\n-\tif (mr->ibv_mr != NULL)\n-\t\tclaim_zero(mlx5_glue->dereg_mr(mr->ibv_mr));\n-\tif (mr->ms_bmp != NULL)\n-\t\trte_bitmap_free(mr->ms_bmp);\n-\trte_free(mr);\n-}\n-\n-/**\n- * Release resources of detached MR having no online entry.\n- *\n- * @param sh\n- * Pointer to Ethernet device shared context.\n- */\n-static void\n-mlx5_mr_garbage_collect(struct mlx5_ibv_shared *sh)\n-{\n-\tstruct mlx5_mr *mr_next;\n-\tstruct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);\n-\n-\t/* Must be called from the primary process. */\n-\tMLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);\n-\t/*\n-\t * MR can't be freed with holding the lock because rte_free() could call\n-\t * memory free callback function. This will be a deadlock situation.\n-\t */\n-\trte_rwlock_write_lock(&sh->mr.rwlock);\n-\t/* Detach the whole free list and release it after unlocking. */\n-\tfree_list = sh->mr.mr_free_list;\n-\tLIST_INIT(&sh->mr.mr_free_list);\n-\trte_rwlock_write_unlock(&sh->mr.rwlock);\n-\t/* Release resources. */\n-\tmr_next = LIST_FIRST(&free_list);\n-\twhile (mr_next != NULL) {\n-\t\tstruct mlx5_mr *mr = mr_next;\n-\n-\t\tmr_next = LIST_NEXT(mr, mr);\n-\t\tmr_free(mr);\n-\t}\n-}\n-\n-/* Called during rte_memseg_contig_walk() by mlx5_mr_create(). */\n-static int\n-mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl,\n-\t\t\t const struct rte_memseg *ms, size_t len, void *arg)\n-{\n-\tstruct mr_find_contig_memsegs_data *data = arg;\n-\n-\tif (data->addr < ms->addr_64 || data->addr >= ms->addr_64 + len)\n-\t\treturn 0;\n-\t/* Found, save it and stop walking. */\n-\tdata->start = ms->addr_64;\n-\tdata->end = ms->addr_64 + len;\n-\tdata->msl = msl;\n-\treturn 1;\n-}\n-\n-/**\n- * Create a new global Memory Region (MR) for a missing virtual address.\n- * This API should be called on a secondary process, then a request is sent to\n- * the primary process in order to create a MR for the address. As the global MR\n- * list is on the shared memory, following LKey lookup should succeed unless the\n- * request fails.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param[out] entry\n- * Pointer to returning MR cache entry, found in the global cache or newly\n- * created. If failed to create one, this will not be updated.\n- * @param addr\n- * Target virtual address to register.\n- *\n- * @return\n- * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.\n- */\n-static uint32_t\n-mlx5_mr_create_secondary(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,\n-\t\t\t uintptr_t addr)\n-{\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n-\tint ret;\n-\n-\tDEBUG(\"port %u requesting MR creation for address (%p)\",\n-\t dev->data->port_id, (void *)addr);\n-\tret = mlx5_mp_req_mr_create(&priv->mp_id, addr);\n-\tif (ret) {\n-\t\tDEBUG(\"port %u fail to request MR creation for address (%p)\",\n-\t\t dev->data->port_id, (void *)addr);\n-\t\treturn UINT32_MAX;\n-\t}\n-\trte_rwlock_read_lock(&priv->sh->mr.rwlock);\n-\t/* Fill in output data. */\n-\tmr_lookup_dev(priv->sh, entry, addr);\n-\t/* Lookup can't fail. */\n-\tMLX5_ASSERT(entry->lkey != UINT32_MAX);\n-\trte_rwlock_read_unlock(&priv->sh->mr.rwlock);\n-\tDEBUG(\"port %u MR CREATED by primary process for %p:\\n\"\n-\t \" [0x%\" PRIxPTR \", 0x%\" PRIxPTR \"), lkey=0x%x\",\n-\t dev->data->port_id, (void *)addr,\n-\t entry->start, entry->end, entry->lkey);\n-\treturn entry->lkey;\n-}\n-\n-/**\n- * Create a new global Memory Region (MR) for a missing virtual address.\n- * Register entire virtually contiguous memory chunk around the address.\n- * This must be called from the primary process.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param[out] entry\n- * Pointer to returning MR cache entry, found in the global cache or newly\n- * created. If failed to create one, this will not be updated.\n- * @param addr\n- * Target virtual address to register.\n- *\n- * @return\n- * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.\n- */\n-uint32_t\n-mlx5_mr_create_primary(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,\n-\t\t uintptr_t addr)\n-{\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n-\tstruct mlx5_ibv_shared *sh = priv->sh;\n-\tstruct mlx5_dev_config *config = &priv->config;\n-\tconst struct rte_memseg_list *msl;\n-\tconst struct rte_memseg *ms;\n-\tstruct mlx5_mr *mr = NULL;\n-\tsize_t len;\n-\tuint32_t ms_n;\n-\tuint32_t bmp_size;\n-\tvoid *bmp_mem;\n-\tint ms_idx_shift = -1;\n-\tunsigned int n;\n-\tstruct mr_find_contig_memsegs_data data = {\n-\t\t.addr = addr,\n-\t};\n-\tstruct mr_find_contig_memsegs_data data_re;\n-\n-\tDRV_LOG(DEBUG, \"port %u creating a MR using address (%p)\",\n-\t\tdev->data->port_id, (void *)addr);\n-\t/*\n-\t * Release detached MRs if any. This can't be called with holding either\n-\t * memory_hotplug_lock or sh->mr.rwlock. MRs on the free list have\n-\t * been detached by the memory free event but it couldn't be released\n-\t * inside the callback due to deadlock. As a result, releasing resources\n-\t * is quite opportunistic.\n-\t */\n-\tmlx5_mr_garbage_collect(sh);\n-\t/*\n-\t * If enabled, find out a contiguous virtual address chunk in use, to\n-\t * which the given address belongs, in order to register maximum range.\n-\t * In the best case where mempools are not dynamically recreated and\n-\t * '--socket-mem' is specified as an EAL option, it is very likely to\n-\t * have only one MR(LKey) per a socket and per a hugepage-size even\n-\t * though the system memory is highly fragmented. As the whole memory\n-\t * chunk will be pinned by kernel, it can't be reused unless entire\n-\t * chunk is freed from EAL.\n-\t *\n-\t * If disabled, just register one memseg (page). Then, memory\n-\t * consumption will be minimized but it may drop performance if there\n-\t * are many MRs to lookup on the datapath.\n-\t */\n-\tif (!config->mr_ext_memseg_en) {\n-\t\tdata.msl = rte_mem_virt2memseg_list((void *)addr);\n-\t\tdata.start = RTE_ALIGN_FLOOR(addr, data.msl->page_sz);\n-\t\tdata.end = data.start + data.msl->page_sz;\n-\t} else if (!rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data)) {\n-\t\tDRV_LOG(WARNING,\n-\t\t\t\"port %u unable to find virtually contiguous\"\n-\t\t\t\" chunk for address (%p).\"\n-\t\t\t\" rte_memseg_contig_walk() failed.\",\n-\t\t\tdev->data->port_id, (void *)addr);\n-\t\trte_errno = ENXIO;\n-\t\tgoto err_nolock;\n-\t}\n-alloc_resources:\n-\t/* Addresses must be page-aligned. */\n-\tMLX5_ASSERT(rte_is_aligned((void *)data.start, data.msl->page_sz));\n-\tMLX5_ASSERT(rte_is_aligned((void *)data.end, data.msl->page_sz));\n-\tmsl = data.msl;\n-\tms = rte_mem_virt2memseg((void *)data.start, msl);\n-\tlen = data.end - data.start;\n-\tMLX5_ASSERT(msl->page_sz == ms->hugepage_sz);\n-\t/* Number of memsegs in the range. */\n-\tms_n = len / msl->page_sz;\n-\tDEBUG(\"port %u extending %p to [0x%\" PRIxPTR \", 0x%\" PRIxPTR \"),\"\n-\t \" page_sz=0x%\" PRIx64 \", ms_n=%u\",\n-\t dev->data->port_id, (void *)addr,\n-\t data.start, data.end, msl->page_sz, ms_n);\n-\t/* Size of memory for bitmap. */\n-\tbmp_size = rte_bitmap_get_memory_footprint(ms_n);\n-\tmr = rte_zmalloc_socket(NULL,\n-\t\t\t\tRTE_ALIGN_CEIL(sizeof(*mr),\n-\t\t\t\t\t RTE_CACHE_LINE_SIZE) +\n-\t\t\t\tbmp_size,\n-\t\t\t\tRTE_CACHE_LINE_SIZE, msl->socket_id);\n-\tif (mr == NULL) {\n-\t\tDEBUG(\"port %u unable to allocate memory for a new MR of\"\n-\t\t \" address (%p).\",\n-\t\t dev->data->port_id, (void *)addr);\n-\t\trte_errno = ENOMEM;\n-\t\tgoto err_nolock;\n-\t}\n-\tmr->msl = msl;\n-\t/*\n-\t * Save the index of the first memseg and initialize memseg bitmap. To\n-\t * see if a memseg of ms_idx in the memseg-list is still valid, check:\n-\t *\trte_bitmap_get(mr->bmp, ms_idx - mr->ms_base_idx)\n-\t */\n-\tmr->ms_base_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);\n-\tbmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);\n-\tmr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);\n-\tif (mr->ms_bmp == NULL) {\n-\t\tDEBUG(\"port %u unable to initialize bitmap for a new MR of\"\n-\t\t \" address (%p).\",\n-\t\t dev->data->port_id, (void *)addr);\n-\t\trte_errno = EINVAL;\n-\t\tgoto err_nolock;\n-\t}\n-\t/*\n-\t * Should recheck whether the extended contiguous chunk is still valid.\n-\t * Because memory_hotplug_lock can't be held if there's any memory\n-\t * related calls in a critical path, resource allocation above can't be\n-\t * locked. If the memory has been changed at this point, try again with\n-\t * just single page. If not, go on with the big chunk atomically from\n-\t * here.\n-\t */\n-\trte_mcfg_mem_read_lock();\n-\tdata_re = data;\n-\tif (len > msl->page_sz &&\n-\t !rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {\n-\t\tDEBUG(\"port %u unable to find virtually contiguous\"\n-\t\t \" chunk for address (%p).\"\n-\t\t \" rte_memseg_contig_walk() failed.\",\n-\t\t dev->data->port_id, (void *)addr);\n-\t\trte_errno = ENXIO;\n-\t\tgoto err_memlock;\n-\t}\n-\tif (data.start != data_re.start || data.end != data_re.end) {\n-\t\t/*\n-\t\t * The extended contiguous chunk has been changed. Try again\n-\t\t * with single memseg instead.\n-\t\t */\n-\t\tdata.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);\n-\t\tdata.end = data.start + msl->page_sz;\n-\t\trte_mcfg_mem_read_unlock();\n-\t\tmr_free(mr);\n-\t\tgoto alloc_resources;\n-\t}\n-\tMLX5_ASSERT(data.msl == data_re.msl);\n-\trte_rwlock_write_lock(&sh->mr.rwlock);\n-\t/*\n-\t * Check the address is really missing. If other thread already created\n-\t * one or it is not found due to overflow, abort and return.\n-\t */\n-\tif (mr_lookup_dev(sh, entry, addr) != UINT32_MAX) {\n-\t\t/*\n-\t\t * Insert to the global cache table. It may fail due to\n-\t\t * low-on-memory. Then, this entry will have to be searched\n-\t\t * here again.\n-\t\t */\n-\t\tmr_btree_insert(&sh->mr.cache, entry);\n-\t\tDEBUG(\"port %u found MR for %p on final lookup, abort\",\n-\t\t dev->data->port_id, (void *)addr);\n-\t\trte_rwlock_write_unlock(&sh->mr.rwlock);\n-\t\trte_mcfg_mem_read_unlock();\n-\t\t/*\n-\t\t * Must be unlocked before calling rte_free() because\n-\t\t * mlx5_mr_mem_event_free_cb() can be called inside.\n-\t\t */\n-\t\tmr_free(mr);\n-\t\treturn entry->lkey;\n-\t}\n-\t/*\n-\t * Trim start and end addresses for verbs MR. Set bits for registering\n-\t * memsegs but exclude already registered ones. Bitmap can be\n-\t * fragmented.\n-\t */\n-\tfor (n = 0; n < ms_n; ++n) {\n-\t\tuintptr_t start;\n-\t\tstruct mlx5_mr_cache ret;\n-\n-\t\tmemset(&ret, 0, sizeof(ret));\n-\t\tstart = data_re.start + n * msl->page_sz;\n-\t\t/* Exclude memsegs already registered by other MRs. */\n-\t\tif (mr_lookup_dev(sh, &ret, start) == UINT32_MAX) {\n-\t\t\t/*\n-\t\t\t * Start from the first unregistered memseg in the\n-\t\t\t * extended range.\n-\t\t\t */\n-\t\t\tif (ms_idx_shift == -1) {\n-\t\t\t\tmr->ms_base_idx += n;\n-\t\t\t\tdata.start = start;\n-\t\t\t\tms_idx_shift = n;\n-\t\t\t}\n-\t\t\tdata.end = start + msl->page_sz;\n-\t\t\trte_bitmap_set(mr->ms_bmp, n - ms_idx_shift);\n-\t\t\t++mr->ms_n;\n-\t\t}\n-\t}\n-\tlen = data.end - data.start;\n-\tmr->ms_bmp_n = len / msl->page_sz;\n-\tMLX5_ASSERT(ms_idx_shift + mr->ms_bmp_n <= ms_n);\n-\t/*\n-\t * Finally create a verbs MR for the memory chunk. ibv_reg_mr() can be\n-\t * called with holding the memory lock because it doesn't use\n-\t * mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()\n-\t * through mlx5_alloc_verbs_buf().\n-\t */\n-\tmr->ibv_mr = mlx5_glue->reg_mr(sh->pd, (void *)data.start, len,\n-\t\t\t\t IBV_ACCESS_LOCAL_WRITE |\n-\t\t\t\t\t IBV_ACCESS_RELAXED_ORDERING);\n-\tif (mr->ibv_mr == NULL) {\n-\t\tDEBUG(\"port %u fail to create a verbs MR for address (%p)\",\n-\t\t dev->data->port_id, (void *)addr);\n-\t\trte_errno = EINVAL;\n-\t\tgoto err_mrlock;\n-\t}\n-\tMLX5_ASSERT((uintptr_t)mr->ibv_mr->addr == data.start);\n-\tMLX5_ASSERT(mr->ibv_mr->length == len);\n-\tLIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);\n-\tDEBUG(\"port %u MR CREATED (%p) for %p:\\n\"\n-\t \" [0x%\" PRIxPTR \", 0x%\" PRIxPTR \"),\"\n-\t \" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u\",\n-\t dev->data->port_id, (void *)mr, (void *)addr,\n-\t data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),\n-\t mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);\n-\t/* Insert to the global cache table. */\n-\tmr_insert_dev_cache(sh, mr);\n-\t/* Fill in output data. */\n-\tmr_lookup_dev(sh, entry, addr);\n-\t/* Lookup can't fail. */\n-\tMLX5_ASSERT(entry->lkey != UINT32_MAX);\n-\trte_rwlock_write_unlock(&sh->mr.rwlock);\n-\trte_mcfg_mem_read_unlock();\n-\treturn entry->lkey;\n-err_mrlock:\n-\trte_rwlock_write_unlock(&sh->mr.rwlock);\n-err_memlock:\n-\trte_mcfg_mem_read_unlock();\n-err_nolock:\n-\t/*\n-\t * In case of error, as this can be called in a datapath, a warning\n-\t * message per an error is preferable instead. Must be unlocked before\n-\t * calling rte_free() because mlx5_mr_mem_event_free_cb() can be called\n-\t * inside.\n-\t */\n-\tmr_free(mr);\n-\treturn UINT32_MAX;\n-}\n-\n-/**\n- * Create a new global Memory Region (MR) for a missing virtual address.\n- * This can be called from primary and secondary process.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param[out] entry\n- * Pointer to returning MR cache entry, found in the global cache or newly\n- * created. If failed to create one, this will not be updated.\n- * @param addr\n- * Target virtual address to register.\n- *\n- * @return\n- * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.\n- */\n-static uint32_t\n-mlx5_mr_create(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,\n-\t uintptr_t addr)\n-{\n-\tuint32_t ret = 0;\n-\n-\tswitch (rte_eal_process_type()) {\n-\tcase RTE_PROC_PRIMARY:\n-\t\tret = mlx5_mr_create_primary(dev, entry, addr);\n-\t\tbreak;\n-\tcase RTE_PROC_SECONDARY:\n-\t\tret = mlx5_mr_create_secondary(dev, entry, addr);\n-\t\tbreak;\n-\tdefault:\n-\t\tbreak;\n-\t}\n-\treturn ret;\n-}\n-\n-/**\n- * Rebuild the global B-tree cache of device from the original MR list.\n- *\n- * @param sh\n- * Pointer to Ethernet device shared context.\n- */\n-static void\n-mr_rebuild_dev_cache(struct mlx5_ibv_shared *sh)\n-{\n-\tstruct mlx5_mr *mr;\n-\n-\tDRV_LOG(DEBUG, \"device %s rebuild dev cache[]\", sh->ibdev_name);\n-\t/* Flush cache to rebuild. */\n-\tsh->mr.cache.len = 1;\n-\tsh->mr.cache.overflow = 0;\n-\t/* Iterate all the existing MRs. */\n-\tLIST_FOREACH(mr, &sh->mr.mr_list, mr)\n-\t\tif (mr_insert_dev_cache(sh, mr) < 0)\n-\t\t\treturn;\n-}\n-\n /**\n * Callback for memory free event. Iterate freed memsegs and check whether it\n * belongs to an existing MR. If found, clear the bit from bitmap of MR. As a\n@@ -900,18 +74,18 @@ mlx5_mr_mem_event_free_cb(struct mlx5_ibv_shared *sh,\n \t\t RTE_ALIGN((uintptr_t)addr, msl->page_sz));\n \tMLX5_ASSERT(len == RTE_ALIGN(len, msl->page_sz));\n \tms_n = len / msl->page_sz;\n-\trte_rwlock_write_lock(&sh->mr.rwlock);\n+\trte_rwlock_write_lock(&sh->share_cache.rwlock);\n \t/* Clear bits of freed memsegs from MR. */\n \tfor (i = 0; i < ms_n; ++i) {\n \t\tconst struct rte_memseg *ms;\n-\t\tstruct mlx5_mr_cache entry;\n+\t\tstruct mr_cache_entry entry;\n \t\tuintptr_t start;\n \t\tint ms_idx;\n \t\tuint32_t pos;\n \n \t\t/* Find MR having this memseg. */\n \t\tstart = (uintptr_t)addr + i * msl->page_sz;\n-\t\tmr = mr_lookup_dev_list(sh, &entry, start);\n+\t\tmr = mlx5_mr_lookup_list(&sh->share_cache, &entry, start);\n \t\tif (mr == NULL)\n \t\t\tcontinue;\n \t\tMLX5_ASSERT(mr->msl); /* Can't be external memory. */\n@@ -927,7 +101,7 @@ mlx5_mr_mem_event_free_cb(struct mlx5_ibv_shared *sh,\n \t\trte_bitmap_clear(mr->ms_bmp, pos);\n \t\tif (--mr->ms_n == 0) {\n \t\t\tLIST_REMOVE(mr, mr);\n-\t\t\tLIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);\n+\t\t\tLIST_INSERT_HEAD(&sh->share_cache.mr_free_list, mr, mr);\n \t\t\tDEBUG(\"device %s remove MR(%p) from list\",\n \t\t\t sh->ibdev_name, (void *)mr);\n \t\t}\n@@ -938,7 +112,7 @@ mlx5_mr_mem_event_free_cb(struct mlx5_ibv_shared *sh,\n \t\trebuild = 1;\n \t}\n \tif (rebuild) {\n-\t\tmr_rebuild_dev_cache(sh);\n+\t\tmlx5_mr_rebuild_cache(&sh->share_cache);\n \t\t/*\n \t\t * Flush local caches by propagating invalidation across cores.\n \t\t * rte_smp_wmb() is enough to synchronize this event. If one of\n@@ -948,12 +122,12 @@ mlx5_mr_mem_event_free_cb(struct mlx5_ibv_shared *sh,\n \t\t * generation below) will be guaranteed to be seen by other core\n \t\t * before the core sees the newly allocated memory.\n \t\t */\n-\t\t++sh->mr.dev_gen;\n+\t\t++sh->share_cache.dev_gen;\n \t\tDEBUG(\"broadcasting local cache flush, gen=%d\",\n-\t\t sh->mr.dev_gen);\n+\t\t sh->share_cache.dev_gen);\n \t\trte_smp_wmb();\n \t}\n-\trte_rwlock_write_unlock(&sh->mr.rwlock);\n+\trte_rwlock_write_unlock(&sh->share_cache.rwlock);\n }\n \n /**\n@@ -990,111 +164,6 @@ mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,\n \t}\n }\n \n-/**\n- * Look up address in the global MR cache table. If not found, create a new MR.\n- * Insert the found/created entry to local bottom-half cache table.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param mr_ctrl\n- * Pointer to per-queue MR control structure.\n- * @param[out] entry\n- * Pointer to returning MR cache entry, found in the global cache or newly\n- * created. If failed to create one, this is not written.\n- * @param addr\n- * Search key.\n- *\n- * @return\n- * Searched LKey on success, UINT32_MAX on no match.\n- */\n-static uint32_t\n-mlx5_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,\n-\t\t struct mlx5_mr_cache *entry, uintptr_t addr)\n-{\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n-\tstruct mlx5_ibv_shared *sh = priv->sh;\n-\tstruct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;\n-\tuint16_t idx;\n-\tuint32_t lkey;\n-\n-\t/* If local cache table is full, try to double it. */\n-\tif (unlikely(bt->len == bt->size))\n-\t\tmr_btree_expand(bt, bt->size << 1);\n-\t/* Look up in the global cache. */\n-\trte_rwlock_read_lock(&sh->mr.rwlock);\n-\tlkey = mr_btree_lookup(&sh->mr.cache, &idx, addr);\n-\tif (lkey != UINT32_MAX) {\n-\t\t/* Found. */\n-\t\t*entry = (*sh->mr.cache.table)[idx];\n-\t\trte_rwlock_read_unlock(&sh->mr.rwlock);\n-\t\t/*\n-\t\t * Update local cache. Even if it fails, return the found entry\n-\t\t * to update top-half cache. Next time, this entry will be found\n-\t\t * in the global cache.\n-\t\t */\n-\t\tmr_btree_insert(bt, entry);\n-\t\treturn lkey;\n-\t}\n-\trte_rwlock_read_unlock(&sh->mr.rwlock);\n-\t/* First time to see the address? Create a new MR. */\n-\tlkey = mlx5_mr_create(dev, entry, addr);\n-\t/*\n-\t * Update the local cache if successfully created a new global MR. Even\n-\t * if failed to create one, there's no action to take in this datapath\n-\t * code. As returning LKey is invalid, this will eventually make HW\n-\t * fail.\n-\t */\n-\tif (lkey != UINT32_MAX)\n-\t\tmr_btree_insert(bt, entry);\n-\treturn lkey;\n-}\n-\n-/**\n- * Bottom-half of LKey search on datapath. Firstly search in cache_bh[] and if\n- * misses, search in the global MR cache table and update the new entry to\n- * per-queue local caches.\n- *\n- * @param dev\n- * Pointer to Ethernet device.\n- * @param mr_ctrl\n- * Pointer to per-queue MR control structure.\n- * @param addr\n- * Search key.\n- *\n- * @return\n- * Searched LKey on success, UINT32_MAX on no match.\n- */\n-static uint32_t\n-mlx5_mr_addr2mr_bh(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,\n-\t\t uintptr_t addr)\n-{\n-\tuint32_t lkey;\n-\tuint16_t bh_idx = 0;\n-\t/* Victim in top-half cache to replace with new entry. */\n-\tstruct mlx5_mr_cache *repl = &mr_ctrl->cache[mr_ctrl->head];\n-\n-\t/* Binary-search MR translation table. */\n-\tlkey = mr_btree_lookup(&mr_ctrl->cache_bh, &bh_idx, addr);\n-\t/* Update top-half cache. */\n-\tif (likely(lkey != UINT32_MAX)) {\n-\t\t*repl = (*mr_ctrl->cache_bh.table)[bh_idx];\n-\t} else {\n-\t\t/*\n-\t\t * If missed in local lookup table, search in the global cache\n-\t\t * and local cache_bh[] will be updated inside if possible.\n-\t\t * Top-half cache entry will also be updated.\n-\t\t */\n-\t\tlkey = mlx5_mr_lookup_dev(dev, mr_ctrl, repl, addr);\n-\t\tif (unlikely(lkey == UINT32_MAX))\n-\t\t\treturn UINT32_MAX;\n-\t}\n-\t/* Update the most recently used entry. */\n-\tmr_ctrl->mru = mr_ctrl->head;\n-\t/* Point to the next victim, the oldest. */\n-\tmr_ctrl->head = (mr_ctrl->head + 1) % MLX5_MR_CACHE_N;\n-\treturn lkey;\n-}\n-\n /**\n * Bottom-half of LKey search on Rx.\n *\n@@ -1114,7 +183,9 @@ mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr)\n \tstruct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;\n \tstruct mlx5_priv *priv = rxq_ctrl->priv;\n \n-\treturn mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr);\n+\treturn mlx5_mr_addr2mr_bh(priv->sh->pd, &priv->mp_id,\n+\t\t\t\t &priv->sh->share_cache, mr_ctrl, addr,\n+\t\t\t\t priv->config.mr_ext_memseg_en);\n }\n \n /**\n@@ -1136,7 +207,9 @@ mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr)\n \tstruct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;\n \tstruct mlx5_priv *priv = txq_ctrl->priv;\n \n-\treturn mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr);\n+\treturn mlx5_mr_addr2mr_bh(priv->sh->pd, &priv->mp_id,\n+\t\t\t\t &priv->sh->share_cache, mr_ctrl, addr,\n+\t\t\t\t priv->config.mr_ext_memseg_en);\n }\n \n /**\n@@ -1165,82 +238,6 @@ mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb)\n \treturn lkey;\n }\n \n-/**\n- * Flush all of the local cache entries.\n- *\n- * @param mr_ctrl\n- * Pointer to per-queue MR control structure.\n- */\n-void\n-mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)\n-{\n-\t/* Reset the most-recently-used index. */\n-\tmr_ctrl->mru = 0;\n-\t/* Reset the linear search array. */\n-\tmr_ctrl->head = 0;\n-\tmemset(mr_ctrl->cache, 0, sizeof(mr_ctrl->cache));\n-\t/* Reset the B-tree table. */\n-\tmr_ctrl->cache_bh.len = 1;\n-\tmr_ctrl->cache_bh.overflow = 0;\n-\t/* Update the generation number. */\n-\tmr_ctrl->cur_gen = *mr_ctrl->dev_gen_ptr;\n-\tDRV_LOG(DEBUG, \"mr_ctrl(%p): flushed, cur_gen=%d\",\n-\t\t(void *)mr_ctrl, mr_ctrl->cur_gen);\n-}\n-\n-/**\n- * Creates a memory region for external memory, that is memory which is not\n- * part of the DPDK memory segments.\n- *\n- * @param dev\n- * Pointer to the ethernet device.\n- * @param addr\n- * Starting virtual address of memory.\n- * @param len\n- * Length of memory segment being mapped.\n- * @param socked_id\n- * Socket to allocate heap memory for the control structures.\n- *\n- * @return\n- * Pointer to MR structure on success, NULL otherwise.\n- */\n-static struct mlx5_mr *\n-mlx5_create_mr_ext(struct rte_eth_dev *dev, uintptr_t addr, size_t len,\n-\t\t int socket_id)\n-{\n-\tstruct mlx5_priv *priv = dev->data->dev_private;\n-\tstruct mlx5_mr *mr = NULL;\n-\n-\tmr = rte_zmalloc_socket(NULL,\n-\t\t\t\tRTE_ALIGN_CEIL(sizeof(*mr),\n-\t\t\t\t\t RTE_CACHE_LINE_SIZE),\n-\t\t\t\tRTE_CACHE_LINE_SIZE, socket_id);\n-\tif (mr == NULL)\n-\t\treturn NULL;\n-\tmr->ibv_mr = mlx5_glue->reg_mr(priv->sh->pd, (void *)addr, len,\n-\t\t\t\t IBV_ACCESS_LOCAL_WRITE |\n-\t\t\t\t\t IBV_ACCESS_RELAXED_ORDERING);\n-\tif (mr->ibv_mr == NULL) {\n-\t\tDRV_LOG(WARNING,\n-\t\t\t\"port %u fail to create a verbs MR for address (%p)\",\n-\t\t\tdev->data->port_id, (void *)addr);\n-\t\trte_free(mr);\n-\t\treturn NULL;\n-\t}\n-\tmr->msl = NULL; /* Mark it is external memory. */\n-\tmr->ms_bmp = NULL;\n-\tmr->ms_n = 1;\n-\tmr->ms_bmp_n = 1;\n-\tDRV_LOG(DEBUG,\n-\t\t\"port %u MR CREATED (%p) for external memory %p:\\n\"\n-\t\t\" [0x%\" PRIxPTR \", 0x%\" PRIxPTR \"),\"\n-\t\t\" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u\",\n-\t\tdev->data->port_id, (void *)mr, (void *)addr,\n-\t\taddr, addr + len, rte_cpu_to_be_32(mr->ibv_mr->lkey),\n-\t\tmr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);\n-\treturn mr;\n-}\n-\n /**\n * Called during rte_mempool_mem_iter() by mlx5_mr_update_ext_mp().\n *\n@@ -1267,19 +264,19 @@ mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,\n \tstruct mlx5_mr *mr = NULL;\n \tuintptr_t addr = (uintptr_t)memhdr->addr;\n \tsize_t len = memhdr->len;\n-\tstruct mlx5_mr_cache entry;\n+\tstruct mr_cache_entry entry;\n \tuint32_t lkey;\n \n \tMLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);\n \t/* If already registered, it should return. */\n-\trte_rwlock_read_lock(&sh->mr.rwlock);\n-\tlkey = mr_lookup_dev(sh, &entry, addr);\n-\trte_rwlock_read_unlock(&sh->mr.rwlock);\n+\trte_rwlock_read_lock(&sh->share_cache.rwlock);\n+\tlkey = mlx5_mr_lookup_cache(&sh->share_cache, &entry, addr);\n+\trte_rwlock_read_unlock(&sh->share_cache.rwlock);\n \tif (lkey != UINT32_MAX)\n \t\treturn;\n \tDRV_LOG(DEBUG, \"port %u register MR for chunk #%d of mempool (%s)\",\n \t\tdev->data->port_id, mem_idx, mp->name);\n-\tmr = mlx5_create_mr_ext(dev, addr, len, mp->socket_id);\n+\tmr = mlx5_create_mr_ext(sh->pd, addr, len, mp->socket_id);\n \tif (!mr) {\n \t\tDRV_LOG(WARNING,\n \t\t\t\"port %u unable to allocate a new MR of\"\n@@ -1288,13 +285,14 @@ mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,\n \t\tdata->ret = -1;\n \t\treturn;\n \t}\n-\trte_rwlock_write_lock(&sh->mr.rwlock);\n-\tLIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);\n+\trte_rwlock_write_lock(&sh->share_cache.rwlock);\n+\tLIST_INSERT_HEAD(&sh->share_cache.mr_list, mr, mr);\n \t/* Insert to the global cache table. */\n-\tmr_insert_dev_cache(sh, mr);\n-\trte_rwlock_write_unlock(&sh->mr.rwlock);\n+\tmlx5_mr_insert_cache(&sh->share_cache, mr);\n+\trte_rwlock_write_unlock(&sh->share_cache.rwlock);\n \t/* Insert to the local cache table */\n-\tmlx5_mr_addr2mr_bh(dev, mr_ctrl, addr);\n+\tmlx5_mr_addr2mr_bh(sh->pd, &priv->mp_id, &sh->share_cache,\n+\t\t\t mr_ctrl, addr, priv->config.mr_ext_memseg_en);\n }\n \n /**\n@@ -1351,19 +349,19 @@ mlx5_dma_map(struct rte_pci_device *pdev, void *addr,\n \t\treturn -1;\n \t}\n \tpriv = dev->data->dev_private;\n-\tmr = mlx5_create_mr_ext(dev, (uintptr_t)addr, len, SOCKET_ID_ANY);\n+\tsh = priv->sh;\n+\tmr = mlx5_create_mr_ext(sh->pd, (uintptr_t)addr, len, SOCKET_ID_ANY);\n \tif (!mr) {\n \t\tDRV_LOG(WARNING,\n \t\t\t\"port %u unable to dma map\", dev->data->port_id);\n \t\trte_errno = EINVAL;\n \t\treturn -1;\n \t}\n-\tsh = priv->sh;\n-\trte_rwlock_write_lock(&sh->mr.rwlock);\n-\tLIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);\n+\trte_rwlock_write_lock(&sh->share_cache.rwlock);\n+\tLIST_INSERT_HEAD(&sh->share_cache.mr_list, mr, mr);\n \t/* Insert to the global cache table. */\n-\tmr_insert_dev_cache(sh, mr);\n-\trte_rwlock_write_unlock(&sh->mr.rwlock);\n+\tmlx5_mr_insert_cache(&sh->share_cache, mr);\n+\trte_rwlock_write_unlock(&sh->share_cache.rwlock);\n \treturn 0;\n }\n \n@@ -1390,7 +388,7 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,\n \tstruct mlx5_priv *priv;\n \tstruct mlx5_ibv_shared *sh;\n \tstruct mlx5_mr *mr;\n-\tstruct mlx5_mr_cache entry;\n+\tstruct mr_cache_entry entry;\n \n \tdev = pci_dev_to_eth_dev(pdev);\n \tif (!dev) {\n@@ -1401,10 +399,10 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,\n \t}\n \tpriv = dev->data->dev_private;\n \tsh = priv->sh;\n-\trte_rwlock_read_lock(&sh->mr.rwlock);\n-\tmr = mr_lookup_dev_list(sh, &entry, (uintptr_t)addr);\n+\trte_rwlock_read_lock(&sh->share_cache.rwlock);\n+\tmr = mlx5_mr_lookup_list(&sh->share_cache, &entry, (uintptr_t)addr);\n \tif (!mr) {\n-\t\trte_rwlock_read_unlock(&sh->mr.rwlock);\n+\t\trte_rwlock_read_unlock(&sh->share_cache.rwlock);\n \t\tDRV_LOG(WARNING, \"address 0x%\" PRIxPTR \" wasn't registered \"\n \t\t\t\t \"to PCI device %p\", (uintptr_t)addr,\n \t\t\t\t (void *)pdev);\n@@ -1412,10 +410,10 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,\n \t\treturn -1;\n \t}\n \tLIST_REMOVE(mr, mr);\n-\tLIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);\n+\tLIST_INSERT_HEAD(&sh->share_cache.mr_free_list, mr, mr);\n \tDEBUG(\"port %u remove MR(%p) from list\", dev->data->port_id,\n \t (void *)mr);\n-\tmr_rebuild_dev_cache(sh);\n+\tmlx5_mr_rebuild_cache(&sh->share_cache);\n \t/*\n \t * Flush local caches by propagating invalidation across cores.\n \t * rte_smp_wmb() is enough to synchronize this event. If one of\n@@ -1425,10 +423,11 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,\n \t * generation below) will be guaranteed to be seen by other core\n \t * before the core sees the newly allocated memory.\n \t */\n-\t++sh->mr.dev_gen;\n-\tDEBUG(\"broadcasting local cache flush, gen=%d\",\tsh->mr.dev_gen);\n+\t++sh->share_cache.dev_gen;\n+\tDEBUG(\"broadcasting local cache flush, gen=%d\",\n+\t sh->share_cache.dev_gen);\n \trte_smp_wmb();\n-\trte_rwlock_read_unlock(&sh->mr.rwlock);\n+\trte_rwlock_read_unlock(&sh->share_cache.rwlock);\n \treturn 0;\n }\n \n@@ -1503,14 +502,19 @@ mlx5_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,\n \t\t unsigned mem_idx __rte_unused)\n {\n \tstruct mr_update_mp_data *data = opaque;\n+\tstruct rte_eth_dev *dev = data->dev;\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\n \tuint32_t lkey;\n \n \t/* Stop iteration if failed in the previous walk. */\n \tif (data->ret < 0)\n \t\treturn;\n \t/* Register address of the chunk and update local caches. */\n-\tlkey = mlx5_mr_addr2mr_bh(data->dev, data->mr_ctrl,\n-\t\t\t\t (uintptr_t)memhdr->addr);\n+\tlkey = mlx5_mr_addr2mr_bh(priv->sh->pd, &priv->mp_id,\n+\t\t\t\t &priv->sh->share_cache, data->mr_ctrl,\n+\t\t\t\t (uintptr_t)memhdr->addr,\n+\t\t\t\t priv->config.mr_ext_memseg_en);\n \tif (lkey == UINT32_MAX)\n \t\tdata->ret = -1;\n }\n@@ -1545,76 +549,3 @@ mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,\n \t}\n \treturn data.ret;\n }\n-\n-/**\n- * Dump all the created MRs and the global cache entries.\n- *\n- * @param sh\n- * Pointer to Ethernet device shared context.\n- */\n-void\n-mlx5_mr_dump_dev(struct mlx5_ibv_shared *sh __rte_unused)\n-{\n-#ifdef RTE_LIBRTE_MLX5_DEBUG\n-\tstruct mlx5_mr *mr;\n-\tint mr_n = 0;\n-\tint chunk_n = 0;\n-\n-\trte_rwlock_read_lock(&sh->mr.rwlock);\n-\t/* Iterate all the existing MRs. */\n-\tLIST_FOREACH(mr, &sh->mr.mr_list, mr) {\n-\t\tunsigned int n;\n-\n-\t\tDEBUG(\"device %s MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u\",\n-\t\t sh->ibdev_name, mr_n++,\n-\t\t rte_cpu_to_be_32(mr->ibv_mr->lkey),\n-\t\t mr->ms_n, mr->ms_bmp_n);\n-\t\tif (mr->ms_n == 0)\n-\t\t\tcontinue;\n-\t\tfor (n = 0; n < mr->ms_bmp_n; ) {\n-\t\t\tstruct mlx5_mr_cache ret = { 0, };\n-\n-\t\t\tn = mr_find_next_chunk(mr, &ret, n);\n-\t\t\tif (!ret.end)\n-\t\t\t\tbreak;\n-\t\t\tDEBUG(\" chunk[%u], [0x%\" PRIxPTR \", 0x%\" PRIxPTR \")\",\n-\t\t\t chunk_n++, ret.start, ret.end);\n-\t\t}\n-\t}\n-\tDEBUG(\"device %s dumping global cache\", sh->ibdev_name);\n-\tmlx5_mr_btree_dump(&sh->mr.cache);\n-\trte_rwlock_read_unlock(&sh->mr.rwlock);\n-#endif\n-}\n-\n-/**\n- * Release all the created MRs and resources for shared device context.\n- * list.\n- *\n- * @param sh\n- * Pointer to Ethernet device shared context.\n- */\n-void\n-mlx5_mr_release(struct mlx5_ibv_shared *sh)\n-{\n-\tstruct mlx5_mr *mr_next;\n-\n-\tif (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG))\n-\t\tmlx5_mr_dump_dev(sh);\n-\trte_rwlock_write_lock(&sh->mr.rwlock);\n-\t/* Detach from MR list and move to free list. */\n-\tmr_next = LIST_FIRST(&sh->mr.mr_list);\n-\twhile (mr_next != NULL) {\n-\t\tstruct mlx5_mr *mr = mr_next;\n-\n-\t\tmr_next = LIST_NEXT(mr, mr);\n-\t\tLIST_REMOVE(mr, mr);\n-\t\tLIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);\n-\t}\n-\tLIST_INIT(&sh->mr.mr_list);\n-\t/* Free global cache. */\n-\tmlx5_mr_btree_free(&sh->mr.cache);\n-\trte_rwlock_write_unlock(&sh->mr.rwlock);\n-\t/* Free all remaining MRs. */\n-\tmlx5_mr_garbage_collect(sh);\n-}\ndiff --git a/drivers/net/mlx5/mlx5_mr.h b/drivers/net/mlx5/mlx5_mr.h\nindex 48264c8294..0c5877b3d6 100644\n--- a/drivers/net/mlx5/mlx5_mr.h\n+++ b/drivers/net/mlx5/mlx5_mr.h\n@@ -24,99 +24,16 @@\n #include <rte_ethdev.h>\n #include <rte_rwlock.h>\n #include <rte_bitmap.h>\n+#include <rte_memory.h>\n \n-/* Memory Region object. */\n-struct mlx5_mr {\n-\tLIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */\n-\tstruct ibv_mr *ibv_mr; /* Verbs Memory Region. */\n-\tconst struct rte_memseg_list *msl;\n-\tint ms_base_idx; /* Start index of msl->memseg_arr[]. */\n-\tint ms_n; /* Number of memsegs in use. */\n-\tuint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */\n-\tstruct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to MR. */\n-};\n-\n-/* Cache entry for Memory Region. */\n-struct mlx5_mr_cache {\n-\tuintptr_t start; /* Start address of MR. */\n-\tuintptr_t end; /* End address of MR. */\n-\tuint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */\n-} __rte_packed;\n-\n-/* MR Cache table for Binary search. */\n-struct mlx5_mr_btree {\n-\tuint16_t len; /* Number of entries. */\n-\tuint16_t size; /* Total number of entries. */\n-\tint overflow; /* Mark failure of table expansion. */\n-\tstruct mlx5_mr_cache (*table)[];\n-} __rte_packed;\n-\n-/* Per-queue MR control descriptor. */\n-struct mlx5_mr_ctrl {\n-\tuint32_t *dev_gen_ptr; /* Generation number of device to poll. */\n-\tuint32_t cur_gen; /* Generation number saved to flush caches. */\n-\tuint16_t mru; /* Index of last hit entry in top-half cache. */\n-\tuint16_t head; /* Index of the oldest entry in top-half cache. */\n-\tstruct mlx5_mr_cache cache[MLX5_MR_CACHE_N]; /* Cache for top-half. */\n-\tstruct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */\n-} __rte_packed;\n-\n-struct mlx5_ibv_shared;\n-extern struct mlx5_dev_list mlx5_mem_event_cb_list;\n-extern rte_rwlock_t mlx5_mem_event_rwlock;\n+#include <mlx5_common_mr.h>\n \n /* First entry must be NULL for comparison. */\n #define mlx5_mr_btree_len(bt) ((bt)->len - 1)\n \n-int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket);\n-void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);\n-uint32_t mlx5_mr_create_primary(struct rte_eth_dev *dev,\n-\t\t\t\tstruct mlx5_mr_cache *entry, uintptr_t addr);\n void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,\n \t\t\t size_t len, void *arg);\n int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,\n \t\t struct rte_mempool *mp);\n-void mlx5_mr_release(struct mlx5_ibv_shared *sh);\n-\n-/* Debug purpose functions. */\n-void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt);\n-void mlx5_mr_dump_dev(struct mlx5_ibv_shared *sh);\n-\n-/**\n- * Look up LKey from given lookup table by linear search. Firstly look up the\n- * last-hit entry. If miss, the entire array is searched. If found, update the\n- * last-hit index and return LKey.\n- *\n- * @param lkp_tbl\n- * Pointer to lookup table.\n- * @param[in,out] cached_idx\n- * Pointer to last-hit index.\n- * @param n\n- * Size of lookup table.\n- * @param addr\n- * Search key.\n- *\n- * @return\n- * Searched LKey on success, UINT32_MAX on no match.\n- */\n-static __rte_always_inline uint32_t\n-mlx5_mr_lookup_cache(struct mlx5_mr_cache *lkp_tbl, uint16_t *cached_idx,\n-\t\t uint16_t n, uintptr_t addr)\n-{\n-\tuint16_t idx;\n-\n-\tif (likely(addr >= lkp_tbl[*cached_idx].start &&\n-\t\t addr < lkp_tbl[*cached_idx].end))\n-\t\treturn lkp_tbl[*cached_idx].lkey;\n-\tfor (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {\n-\t\tif (addr >= lkp_tbl[idx].start &&\n-\t\t addr < lkp_tbl[idx].end) {\n-\t\t\t/* Found. */\n-\t\t\t*cached_idx = idx;\n-\t\t\treturn lkp_tbl[idx].lkey;\n-\t\t}\n-\t}\n-\treturn UINT32_MAX;\n-}\n \n #endif /* RTE_PMD_MLX5_MR_H_ */\ndiff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c\nindex fc7591c2b0..5f9b670442 100644\n--- a/drivers/net/mlx5/mlx5_rxtx.c\n+++ b/drivers/net/mlx5/mlx5_rxtx.c\n@@ -33,6 +33,7 @@\n \n #include \"mlx5_defs.h\"\n #include \"mlx5.h\"\n+#include \"mlx5_mr.h\"\n #include \"mlx5_utils.h\"\n #include \"mlx5_rxtx.h\"\n #include \"mlx5_autoconf.h\"\ndiff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h\nindex 939778aa55..84161ad6af 100644\n--- a/drivers/net/mlx5/mlx5_rxtx.h\n+++ b/drivers/net/mlx5/mlx5_rxtx.h\n@@ -34,11 +34,11 @@\n #include <mlx5_glue.h>\n #include <mlx5_prm.h>\n #include <mlx5_common.h>\n+#include <mlx5_common_mr.h>\n \n #include \"mlx5_defs.h\"\n #include \"mlx5_utils.h\"\n #include \"mlx5.h\"\n-#include \"mlx5_mr.h\"\n #include \"mlx5_autoconf.h\"\n \n /* Support tunnel matching. */\n@@ -598,8 +598,8 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)\n \tuint32_t lkey;\n \n \t/* Linear search on MR cache array. */\n-\tlkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,\n-\t\t\t\t MLX5_MR_CACHE_N, addr);\n+\tlkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,\n+\t\t\t\t MLX5_MR_CACHE_N, addr);\n \tif (likely(lkey != UINT32_MAX))\n \t\treturn lkey;\n \t/* Take slower bottom-half (Binary Search) on miss. */\n@@ -630,8 +630,8 @@ mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)\n \tif (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))\n \t\tmlx5_mr_flush_local_cache(mr_ctrl);\n \t/* Linear search on MR cache array. */\n-\tlkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,\n-\t\t\t\t MLX5_MR_CACHE_N, addr);\n+\tlkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,\n+\t\t\t\t MLX5_MR_CACHE_N, addr);\n \tif (likely(lkey != UINT32_MAX))\n \t\treturn lkey;\n \t/* Take slower bottom-half on miss. */\ndiff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h\nindex ea925156f0..6ddcbfb0ad 100644\n--- a/drivers/net/mlx5/mlx5_rxtx_vec.h\n+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h\n@@ -13,6 +13,8 @@\n \n #include \"mlx5_autoconf.h\"\n \n+#include \"mlx5_mr.h\"\n+\n /* HW checksum offload capabilities of vectorized Tx. */\n #define MLX5_VEC_TX_CKSUM_OFFLOAD_CAP \\\n \t(DEV_TX_OFFLOAD_IPV4_CKSUM | \\\ndiff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c\nindex 438b705952..759670408b 100644\n--- a/drivers/net/mlx5/mlx5_trigger.c\n+++ b/drivers/net/mlx5/mlx5_trigger.c\n@@ -11,6 +11,7 @@\n #include <rte_alarm.h>\n \n #include \"mlx5.h\"\n+#include \"mlx5_mr.h\"\n #include \"mlx5_rxtx.h\"\n #include \"mlx5_utils.h\"\n #include \"rte_pmd_mlx5.h\"\ndiff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c\nindex 0653f4cf30..29e5cabab6 100644\n--- a/drivers/net/mlx5/mlx5_txq.c\n+++ b/drivers/net/mlx5/mlx5_txq.c\n@@ -30,6 +30,7 @@\n #include <mlx5_glue.h>\n #include <mlx5_devx_cmds.h>\n #include <mlx5_common.h>\n+#include <mlx5_common_mr.h>\n \n #include \"mlx5_defs.h\"\n #include \"mlx5_utils.h\"\n@@ -1289,7 +1290,7 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\tgoto error;\n \t}\n \t/* Save pointer of global generation number to check memory event. */\n-\ttmpl->txq.mr_ctrl.dev_gen_ptr = &priv->sh->mr.dev_gen;\n+\ttmpl->txq.mr_ctrl.dev_gen_ptr = &priv->sh->share_cache.dev_gen;\n \tMLX5_ASSERT(desc > MLX5_TX_COMP_THRESH);\n \ttmpl->txq.offloads = conf->offloads |\n \t\t\t dev->data->dev_conf.txmode.offloads;\n", "prefixes": [ "v3", "4/4" ] }