Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/51672/?format=api
{ "id": 51672, "url": "http://patches.dpdk.org/api/patches/51672/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20190325193627.19726-4-yskoh@mellanox.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20190325193627.19726-4-yskoh@mellanox.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20190325193627.19726-4-yskoh@mellanox.com", "date": "2019-03-25T19:36:27", "name": "[3/3] net/mlx4: remove device register remap", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "ba4e13ef2b59d3e760e3e0b0db5ac6d261272301", "submitter": { "id": 636, "url": "http://patches.dpdk.org/api/people/636/?format=api", "name": "Yongseok Koh", "email": "yskoh@mellanox.com" }, "delegate": { "id": 6624, "url": "http://patches.dpdk.org/api/users/6624/?format=api", "username": "shahafs", "first_name": "Shahaf", "last_name": "Shuler", "email": "shahafs@mellanox.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20190325193627.19726-4-yskoh@mellanox.com/mbox/", "series": [ { "id": 3908, "url": "http://patches.dpdk.org/api/series/3908/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=3908", "date": "2019-03-25T19:36:25", "name": "net/mlx: remove device register remap", "version": 1, "mbox": "http://patches.dpdk.org/series/3908/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/51672/comments/", "check": "fail", "checks": "http://patches.dpdk.org/api/patches/51672/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 8E4A14F9A;\n\tMon, 25 Mar 2019 20:36:45 +0100 (CET)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n\tby dpdk.org (Postfix) with ESMTP id E52AD4C9C\n\tfor <dev@dpdk.org>; Mon, 25 Mar 2019 20:36:39 +0100 (CET)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n\tyskoh@mellanox.com)\n\twith ESMTPS (AES256-SHA encrypted); 25 Mar 2019 21:36:35 +0200", "from scfae-sc-2.mti.labs.mlnx (scfae-sc-2.mti.labs.mlnx\n\t[10.101.0.96])\n\tby labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2PJaTFI024575;\n\tMon, 25 Mar 2019 21:36:34 +0200" ], "From": "Yongseok Koh <yskoh@mellanox.com>", "To": "shahafs@mellanox.com", "Cc": "dev@dpdk.org", "Date": "Mon, 25 Mar 2019 12:36:27 -0700", "Message-Id": "<20190325193627.19726-4-yskoh@mellanox.com>", "X-Mailer": "git-send-email 2.11.0", "In-Reply-To": "<20190325193627.19726-1-yskoh@mellanox.com>", "References": "<20190325193627.19726-1-yskoh@mellanox.com>", "Subject": "[dpdk-dev] [PATCH 3/3] net/mlx4: remove device register remap", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "UAR (User Access Region) registers will be stored in a process-local table\nand a process accesses a register in a table entry with index. Alloc/free\nof table entry is managed by a global bitmap.\n\nWhen there's a need to store a UAR register such as Tx BlueFlame register\nfor doorbell, an index should be allocated by mlx4_uar_alloc_index() and\naddress of the allocated table entry must be acquired by\nmlx4_uar_get_addr_ptr() so that the table can be expanded if overflowed.\nThe local UAR register table doesn't cover all the indexes in the bitmap.\nThis will be expanded if more indexes are allocated than the current size\nof the table.\n\nFor example, the BlueFlame register for Tx doorbell has to be remapped on\neach secondary process. On initialization, primary process allocates an\nindex for the UAR register table and stores the register address in the\nindexed entry of its own table when configuring a Tx queue. The index is\nstored in the shared memory(txq->bfreg_idx) and visiable to secondary\nprocesses. As secondary processes know the index, each process stores\nremapped register in the same indexed entry of its local UAR register\ntable.\n\nOn the datapath of each process, the register can be referenced simply by\nMLX4_UAR_REG(idx) which accesses its local UAR register table by the index.\n\nSigned-off-by: Yongseok Koh <yskoh@mellanox.com>\n---\n drivers/net/mlx4/mlx4.c | 274 ++++++++++++++++++++++++++++---------------\n drivers/net/mlx4/mlx4.h | 22 +++-\n drivers/net/mlx4/mlx4_prm.h | 2 -\n drivers/net/mlx4/mlx4_rxtx.c | 2 +-\n drivers/net/mlx4/mlx4_rxtx.h | 3 +-\n drivers/net/mlx4/mlx4_txq.c | 102 ++++++----------\n 6 files changed, 235 insertions(+), 170 deletions(-)", "diff": "diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c\nindex d913c2a47e..7749e0f9e4 100644\n--- a/drivers/net/mlx4/mlx4.c\n+++ b/drivers/net/mlx4/mlx4.c\n@@ -63,7 +63,7 @@ struct mlx4_shared_data *mlx4_shared_data;\n static rte_spinlock_t mlx4_shared_data_lock = RTE_SPINLOCK_INITIALIZER;\n \n /* Process local data for secondary processes. */\n-static struct mlx4_local_data mlx4_local_data;\n+struct mlx4_local_data mlx4_local_data;\n \n /** Configuration structure for device arguments. */\n struct mlx4_conf {\n@@ -267,11 +267,6 @@ mlx4_dev_start(struct rte_eth_dev *dev)\n \t\treturn 0;\n \tDEBUG(\"%p: attaching configured flows to all RX queues\", (void *)dev);\n \tpriv->started = 1;\n-\tret = mlx4_tx_uar_remap(dev, priv->ctx->cmd_fd);\n-\tif (ret) {\n-\t\tERROR(\"%p: cannot remap UAR\", (void *)dev);\n-\t\tgoto err;\n-\t}\n \tret = mlx4_rss_init(priv);\n \tif (ret) {\n \t\tERROR(\"%p: cannot initialize RSS resources: %s\",\n@@ -319,8 +314,6 @@ static void\n mlx4_dev_stop(struct rte_eth_dev *dev)\n {\n \tstruct mlx4_priv *priv = dev->data->dev_private;\n-\tconst size_t page_size = sysconf(_SC_PAGESIZE);\n-\tint i;\n \n \tif (!priv->started)\n \t\treturn;\n@@ -334,15 +327,6 @@ mlx4_dev_stop(struct rte_eth_dev *dev)\n \tmlx4_flow_sync(priv, NULL);\n \tmlx4_rxq_intr_disable(priv);\n \tmlx4_rss_deinit(priv);\n-\tfor (i = 0; i != dev->data->nb_tx_queues; ++i) {\n-\t\tstruct txq *txq;\n-\n-\t\ttxq = dev->data->tx_queues[i];\n-\t\tif (!txq)\n-\t\t\tcontinue;\n-\t\tmunmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq->msq.db,\n-\t\t\t\t\t page_size), page_size);\n-\t}\n }\n \n /**\n@@ -669,128 +653,224 @@ mlx4_hw_rss_sup(struct ibv_context *ctx, struct ibv_pd *pd,\n \n static struct rte_pci_driver mlx4_driver;\n \n+/**\n+ * Expand the local UAR register table.\n+ *\n+ * @param size\n+ * Size of the table to be expanded\n+ *\n+ * @return\n+ * 0 on success, a negative errno value otherwise and rte_errno is set.\n+ */\n static int\n-find_lower_va_bound(const struct rte_memseg_list *msl,\n-\t\tconst struct rte_memseg *ms, void *arg)\n+uar_expand_table(uint32_t size)\n {\n-\tvoid **addr = arg;\n+\tstruct mlx4_local_data *ld = &mlx4_local_data;\n+\tvoid *mem;\n+\tsize_t tbl_sz = ld->uar_table_sz;\n \n-\tif (msl->external)\n+\tif (size <= tbl_sz)\n \t\treturn 0;\n-\tif (*addr == NULL)\n-\t\t*addr = ms->addr;\n-\telse\n-\t\t*addr = RTE_MIN(*addr, ms->addr);\n-\n+\ttbl_sz = RTE_ALIGN_CEIL(size, RTE_BITMAP_SLAB_BIT_SIZE);\n+\tmem = rte_realloc(ld->uar_table, tbl_sz * sizeof(void *),\n+\t\t\t RTE_CACHE_LINE_SIZE);\n+\tif (!mem) {\n+\t\trte_errno = ENOMEM;\n+\t\tERROR(\"failed to expand uar table\");\n+\t\treturn -rte_errno;\n+\t}\n+\tDEBUG(\"UAR reg. table is expanded to %zu\", tbl_sz);\n+\tld->uar_table = mem;\n+\tld->uar_table_sz = tbl_sz;\n \treturn 0;\n }\n \n /**\n- * Reserve UAR address space for primary process.\n+ * Return the pointer of the indexed slot in the local UAR register table.\n *\n- * Process local resource is used by both primary and secondary to avoid\n- * duplicate reservation. The space has to be available on both primary and\n- * secondary process, TXQ UAR maps to this area using fixed mmap w/o double\n- * check.\n+ * The indexed slot must be allocated by mlx4_uar_alloc_index() in advance. And\n+ * the table will be expanded if overflowed.\n+ *\n+ * @param idx\n+ * Index of the table.\n *\n * @return\n- * 0 on success, a negative errno value otherwise and rte_errno is set.\n+ * Pointer of table entry on success, NULL otherwise and rte_errno is set.\n */\n-static int\n-mlx4_uar_init_primary(void)\n+void **\n+mlx4_uar_get_addr_ptr(uint32_t idx)\n+{\n+\tstruct mlx4_local_data *ld = &mlx4_local_data;\n+\tint ret;\n+\n+\tassert(idx < MLX4_UAR_TABLE_SIZE_MAX);\n+\tif (idx >= ld->uar_table_sz) {\n+\t\tret = uar_expand_table(idx + 1);\n+\t\tif (ret)\n+\t\t\treturn NULL;\n+\t}\n+\treturn &(*ld->uar_table)[idx];\n+}\n+\n+/**\n+ * Allocate a slot of UAR register table.\n+ *\n+ * Allocation is done by scanning the global bitmap. The global spinlock should\n+ * be held.\n+ *\n+ * @return\n+ * Index of a free slot on success, a negative errno value otherwise and\n+ * rte_errno is set.\n+ */\n+uint32_t\n+mlx4_uar_alloc_index(void)\n {\n \tstruct mlx4_shared_data *sd = mlx4_shared_data;\n-\tvoid *addr = (void *)0;\n+\tuint32_t idx = 0;\n+\tuint64_t slab = 0;\n+\tint ret;\n \n-\tif (sd->uar_base)\n-\t\treturn 0;\n-\t/* find out lower bound of hugepage segments */\n-\trte_memseg_walk(find_lower_va_bound, &addr);\n-\t/* keep distance to hugepages to minimize potential conflicts. */\n-\taddr = RTE_PTR_SUB(addr, (uintptr_t)(MLX4_UAR_OFFSET + MLX4_UAR_SIZE));\n-\t/* anonymous mmap, no real memory consumption. */\n-\taddr = mmap(addr, MLX4_UAR_SIZE,\n-\t\t PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);\n-\tif (addr == MAP_FAILED) {\n-\t\tERROR(\"failed to reserve UAR address space, please\"\n-\t\t \" adjust MLX4_UAR_SIZE or try --base-virtaddr\");\n-\t\trte_errno = ENOMEM;\n+\tassert(rte_eal_process_type() == RTE_PROC_PRIMARY);\n+\trte_spinlock_lock(&sd->lock);\n+\t__rte_bitmap_scan_init(sd->uar_bmp);\n+\tret = rte_bitmap_scan(sd->uar_bmp, &idx, &slab);\n+\tif (unlikely(!ret)) {\n+\t\t/*\n+\t\t * This cannot happen unless there are unreasonably large number\n+\t\t * of queues and ports.\n+\t\t */\n+\t\trte_errno = ENOSPC;\n+\t\trte_spinlock_unlock(&sd->lock);\n \t\treturn -rte_errno;\n \t}\n-\t/* Accept either same addr or a new addr returned from mmap if target\n-\t * range occupied.\n-\t */\n-\tINFO(\"reserved UAR address space: %p\", addr);\n-\tsd->uar_base = addr; /* for primary and secondary UAR re-mmap. */\n-\treturn 0;\n+\tidx += __builtin_ctzll(slab);\n+\t/* Mark the slot is occupied. */\n+\trte_bitmap_clear(sd->uar_bmp, idx);\n+\trte_spinlock_unlock(&sd->lock);\n+\tDEBUG(\"index %d is allocated in UAR reg. table\", idx);\n+\treturn idx;\n }\n \n /**\n- * Unmap UAR address space reserved for primary process.\n+ * Free a slot of UAR register table.\n */\n-static void\n-mlx4_uar_uninit_primary(void)\n+void\n+mlx4_uar_free_index(uint32_t idx)\n {\n \tstruct mlx4_shared_data *sd = mlx4_shared_data;\n \n-\tif (!sd->uar_base)\n-\t\treturn;\n-\tmunmap(sd->uar_base, MLX4_UAR_SIZE);\n-\tsd->uar_base = NULL;\n+\tassert(rte_eal_process_type() == RTE_PROC_PRIMARY);\n+\tassert(idx < MLX4_UAR_TABLE_SIZE_MAX);\n+\trte_spinlock_lock(&sd->lock);\n+\t/* Mark the slot is empty. */\n+\trte_bitmap_set(sd->uar_bmp, idx);\n+\trte_spinlock_unlock(&sd->lock);\n+\tDEBUG(\"index %d is freed in UAR reg. table\", idx);\n }\n \n /**\n- * Reserve UAR address space for secondary process, align with primary process.\n+ * Initialize UAR register table bitmap.\n+ *\n+ * UAR registers will be stored in a process-local table and the table is\n+ * managed by a global bitmap. When there's a need to store a UAR register, an\n+ * index should be allocated by mlx4_uar_alloc_index() and address of the\n+ * allocated table entry must be acquired by mlx4_uar_get_addr_ptr() so that the\n+ * table can be expanded if overflowed.\n+ *\n+ * The local UAR register table doesn't cover all the indexes in the bitmap.\n+ * This will be expanded if more indexes are allocated than the current size of\n+ * the table.\n+ *\n+ * Secondary process should have reference of the index and store remapped\n+ * register at the same index in its local UAR register table.\n+ *\n+ * On the datapath of each process, the register can be referenced simply by\n+ * MLX4_UAR_REG(idx).\n *\n * @return\n * 0 on success, a negative errno value otherwise and rte_errno is set.\n */\n static int\n-mlx4_uar_init_secondary(void)\n+uar_init_primary(void)\n {\n \tstruct mlx4_shared_data *sd = mlx4_shared_data;\n-\tstruct mlx4_local_data *ld = &mlx4_local_data;\n-\tvoid *addr;\n+\tstruct rte_bitmap *bmp;\n+\tvoid *bmp_mem;\n+\tuint32_t bmp_size;\n+\tunsigned int i;\n \n-\tif (ld->uar_base) { /* Already reserved. */\n-\t\tassert(sd->uar_base == ld->uar_base);\n-\t\treturn 0;\n-\t}\n-\tassert(sd->uar_base);\n-\t/* anonymous mmap, no real memory consumption. */\n-\taddr = mmap(sd->uar_base, MLX4_UAR_SIZE,\n-\t\t PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);\n-\tif (addr == MAP_FAILED) {\n-\t\tERROR(\"UAR mmap failed: %p size: %llu\",\n-\t\t sd->uar_base, MLX4_UAR_SIZE);\n-\t\trte_errno = ENXIO;\n+\tbmp_size = rte_bitmap_get_memory_footprint(MLX4_UAR_TABLE_SIZE_MAX);\n+\tbmp_mem = rte_zmalloc(\"uar_table\", bmp_size, RTE_CACHE_LINE_SIZE);\n+\tif (!bmp_mem) {\n+\t\trte_errno = ENOMEM;\n+\t\tERROR(\"failed to allocate memory for uar table\");\n \t\treturn -rte_errno;\n \t}\n-\tif (sd->uar_base != addr) {\n-\t\tERROR(\"UAR address %p size %llu occupied, please\"\n-\t\t \" adjust MLX4_UAR_OFFSET or try EAL parameter\"\n-\t\t \" --base-virtaddr\",\n-\t\t sd->uar_base, MLX4_UAR_SIZE);\n-\t\trte_errno = ENXIO;\n-\t\treturn -rte_errno;\n+\tbmp = rte_bitmap_init(MLX4_UAR_TABLE_SIZE_MAX, bmp_mem, bmp_size);\n+\t/* Set the entire bitmap as 1 means vacant and 0 means empty. */\n+\tfor (i = 0; i < bmp->array2_size; ++i)\n+\t\trte_bitmap_set_slab(bmp, i * RTE_BITMAP_SLAB_BIT_SIZE, -1);\n+\tsd->uar_bmp = bmp;\n+\treturn 0;\n+}\n+\n+/**\n+ * Un-initialize UAR register resources.\n+ *\n+ * The global bitmap and the register table of primary process are freed.\n+ */\n+static void\n+uar_uninit_primary(void)\n+{\n+\tstruct mlx4_shared_data *sd = mlx4_shared_data;\n+\tstruct mlx4_local_data *ld = &mlx4_local_data;\n+\n+\tif (sd->uar_bmp) {\n+\t\trte_bitmap_free(sd->uar_bmp);\n+\t\trte_free(sd->uar_bmp);\n+\t\tsd->uar_bmp = NULL;\n+\t}\n+\t/* Free primary's table. */\n+\tif (ld->uar_table) {\n+\t\trte_free(ld->uar_table);\n+\t\tld->uar_table = NULL;\n+\t\tld->uar_table_sz = 0;\n \t}\n-\tld->uar_base = addr;\n-\tINFO(\"reserved UAR address space: %p\", addr);\n+}\n+\n+/**\n+ * Initialize UAR register resources for secondary process.\n+ *\n+ * Allocate the local UAR register table. Initially, the number of entries is\n+ * same as the size of a bitmap slab.\n+ *\n+ * @return\n+ * 0 on success, a negative errno value otherwise and rte_errno is set.\n+ */\n+static int\n+uar_init_secondary(void)\n+{\n+\t/* Prepare at least a bitmap slab. */\n+\tuar_expand_table(RTE_BITMAP_SLAB_BIT_SIZE);\n \treturn 0;\n }\n \n /**\n- * Unmap UAR address space reserved for secondary process.\n+ * Un-initialize UAR register resources for secondary process.\n+ *\n+ * The local UAR register table is freed.\n */\n static void\n-mlx4_uar_uninit_secondary(void)\n+uar_uninit_secondary(void)\n {\n \tstruct mlx4_local_data *ld = &mlx4_local_data;\n \n-\tif (!ld->uar_base)\n-\t\treturn;\n-\tmunmap(ld->uar_base, MLX4_UAR_SIZE);\n-\tld->uar_base = NULL;\n+\t/* Free process-local table. */\n+\tif (ld->uar_table) {\n+\t\trte_free(ld->uar_table);\n+\t\tld->uar_table = NULL;\n+\t\tld->uar_table_sz = 0;\n+\t}\n }\n \n /**\n@@ -824,7 +904,7 @@ mlx4_init_once(void)\n \t\trte_mem_event_callback_register(\"MLX4_MEM_EVENT_CB\",\n \t\t\t\t\t\tmlx4_mr_mem_event_cb, NULL);\n \t\tmlx4_mp_init_primary();\n-\t\tret = mlx4_uar_init_primary();\n+\t\tret = uar_init_primary();\n \t\tif (ret)\n \t\t\tgoto error;\n \t\tsd->init_done = true;\n@@ -833,7 +913,7 @@ mlx4_init_once(void)\n \t\tif (ld->init_done)\n \t\t\tbreak;\n \t\tmlx4_mp_init_secondary();\n-\t\tret = mlx4_uar_init_secondary();\n+\t\tret = uar_init_secondary();\n \t\tif (ret)\n \t\t\tgoto error;\n \t\t++sd->secondary_cnt;\n@@ -847,12 +927,12 @@ mlx4_init_once(void)\n error:\n \tswitch (rte_eal_process_type()) {\n \tcase RTE_PROC_PRIMARY:\n-\t\tmlx4_uar_uninit_primary();\n+\t\tuar_uninit_primary();\n \t\tmlx4_mp_uninit_primary();\n \t\trte_mem_event_callback_unregister(\"MLX4_MEM_EVENT_CB\", NULL);\n \t\tbreak;\n \tcase RTE_PROC_SECONDARY:\n-\t\tmlx4_uar_uninit_secondary();\n+\t\tuar_uninit_secondary();\n \t\tmlx4_mp_uninit_secondary();\n \t\tbreak;\n \tdefault:\n@@ -1011,7 +1091,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)\n \t\t\t\tgoto error;\n \t\t\t}\n \t\t\t/* Remap UAR for Tx queues. */\n-\t\t\terr = mlx4_tx_uar_remap(eth_dev, err);\n+\t\t\terr = mlx4_txq_uar_init_secondary(eth_dev, err);\n \t\t\tif (err) {\n \t\t\t\terr = rte_errno;\n \t\t\t\tgoto error;\ndiff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h\nindex 3881943ef0..977866e017 100644\n--- a/drivers/net/mlx4/mlx4.h\n+++ b/drivers/net/mlx4/mlx4.h\n@@ -137,8 +137,8 @@ struct mlx4_shared_data {\n \t/* Global spinlock for primary and secondary processes. */\n \tint init_done; /* Whether primary has done initialization. */\n \tunsigned int secondary_cnt; /* Number of secondary processes init'd. */\n-\tvoid *uar_base;\n-\t/* Reserved UAR address space for TXQ UAR(hw doorbell) mapping. */\n+\tstruct rte_bitmap *uar_bmp;\n+\t/* Bitmap to keep track of BlueFlame register table. */\n \tstruct mlx4_dev_list mem_event_cb_list;\n \trte_rwlock_t mem_event_rwlock;\n };\n@@ -146,11 +146,19 @@ struct mlx4_shared_data {\n /* Per-process data structure, not visible to other processes. */\n struct mlx4_local_data {\n \tint init_done; /* Whether a secondary has done initialization. */\n-\tvoid *uar_base;\n-\t/* Reserved UAR address space for TXQ UAR(hw doorbell) mapping. */\n+\tvoid *(*uar_table)[];\n+\t/* Table of BlueFlame registers for each process. */\n+\tsize_t uar_table_sz;\n+\t/* Size of BlueFlame register table. */\n };\n \n extern struct mlx4_shared_data *mlx4_shared_data;\n+extern struct mlx4_local_data mlx4_local_data;\n+\n+/* The maximum size of BlueFlame register table. */\n+#define MLX4_UAR_TABLE_SIZE_MAX (RTE_MAX_ETHPORTS * RTE_MAX_QUEUES_PER_PORT)\n+\n+#define MLX4_UAR_REG(idx) ((*mlx4_local_data.uar_table)[(idx)])\n \n /** Private data structure. */\n struct mlx4_priv {\n@@ -197,6 +205,12 @@ struct mlx4_priv {\n #define PORT_ID(priv) ((priv)->dev_data->port_id)\n #define ETH_DEV(priv) (&rte_eth_devices[PORT_ID(priv)])\n \n+/* mlx4.c */\n+\n+void **mlx4_uar_get_addr_ptr(uint32_t idx);\n+uint32_t mlx4_uar_alloc_index(void);\n+void mlx4_uar_free_index(uint32_t idx);\n+\n /* mlx4_ethdev.c */\n \n int mlx4_get_ifname(const struct mlx4_priv *priv, char (*ifname)[IF_NAMESIZE]);\ndiff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h\nindex b3e11dde25..06ad92d391 100644\n--- a/drivers/net/mlx4/mlx4_prm.h\n+++ b/drivers/net/mlx4/mlx4_prm.h\n@@ -77,8 +77,6 @@ struct mlx4_sq {\n \tuint32_t owner_opcode;\n \t/**< Default owner opcode with HW valid owner bit. */\n \tuint32_t stamp; /**< Stamp value with an invalid HW owner bit. */\n-\tvolatile uint32_t *qp_sdb; /**< Pointer to the doorbell. */\n-\tvolatile uint32_t *db; /**< Pointer to the doorbell remapped. */\n \toff_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */\n \tuint32_t doorbell_qpn; /**< qp number to write to the doorbell. */\n };\ndiff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c\nindex f22f1ba559..513c8a61bf 100644\n--- a/drivers/net/mlx4/mlx4_rxtx.c\n+++ b/drivers/net/mlx4/mlx4_rxtx.c\n@@ -1048,7 +1048,7 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)\n \t/* Make sure that descriptors are written before doorbell record. */\n \trte_wmb();\n \t/* Ring QP doorbell. */\n-\trte_write32(txq->msq.doorbell_qpn, txq->msq.db);\n+\trte_write32(txq->msq.doorbell_qpn, MLX4_UAR_REG(txq->bfreg_idx));\n \ttxq->elts_head += i;\n \treturn i;\n }\ndiff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h\nindex 7d7a8988ed..d9df98715e 100644\n--- a/drivers/net/mlx4/mlx4_rxtx.h\n+++ b/drivers/net/mlx4/mlx4_rxtx.h\n@@ -97,6 +97,7 @@ struct mlx4_txq_stats {\n struct txq {\n \tstruct mlx4_sq msq; /**< Info for directly manipulating the SQ. */\n \tstruct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */\n+\tuint32_t bfreg_idx; /**< Blueflame register index. */\n \tunsigned int elts_head; /**< Current index in (*elts)[]. */\n \tunsigned int elts_tail; /**< First element awaiting completion. */\n \tint elts_comp_cd; /**< Countdown for next completion. */\n@@ -152,7 +153,7 @@ uint16_t mlx4_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,\n \n /* mlx4_txq.c */\n \n-int mlx4_tx_uar_remap(struct rte_eth_dev *dev, int fd);\n+int mlx4_txq_uar_init_secondary(struct rte_eth_dev *dev, int fd);\n uint64_t mlx4_get_tx_port_offloads(struct mlx4_priv *priv);\n int mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx,\n \t\t\tuint16_t desc, unsigned int socket,\ndiff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c\nindex ed00843425..51d74d6c80 100644\n--- a/drivers/net/mlx4/mlx4_txq.c\n+++ b/drivers/net/mlx4/mlx4_txq.c\n@@ -39,10 +39,15 @@\n #include \"mlx4_rxtx.h\"\n #include \"mlx4_utils.h\"\n \n+#ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET\n /**\n- * Mmap TX UAR(HW doorbell) pages into reserved UAR address space.\n- * Both primary and secondary process do mmap to make UAR address\n- * aligned.\n+ * Initialize UAR register access for Tx.\n+ *\n+ * Primary process shouldn't call this function.\n+ *\n+ * For secondary, remap BlueFlame registers for secondary process. Remapped\n+ * address is stored at the same indexed entry of the local UAR register table\n+ * as primary process.\n *\n * @param[in] dev\n * Pointer to Ethernet device.\n@@ -52,83 +57,41 @@\n * @return\n * 0 on success, a negative errno value otherwise and rte_errno is set.\n */\n-#ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET\n int\n-mlx4_tx_uar_remap(struct rte_eth_dev *dev, int fd)\n+mlx4_txq_uar_init_secondary(struct rte_eth_dev *dev, int fd)\n {\n-\tunsigned int i, j;\n \tconst unsigned int txqs_n = dev->data->nb_tx_queues;\n-\tuintptr_t pages[txqs_n];\n-\tunsigned int pages_n = 0;\n-\tuintptr_t uar_va;\n-\tuintptr_t off;\n-\tvoid *addr;\n-\tvoid *ret;\n \tstruct txq *txq;\n-\tint already_mapped;\n+\tvoid *addr;\n+\tvoid **addr_ptr;\n \tsize_t page_size = sysconf(_SC_PAGESIZE);\n+\tunsigned int i;\n \n-\tmemset(pages, 0, txqs_n * sizeof(uintptr_t));\n+\tassert(rte_eal_process_type() == RTE_PROC_SECONDARY);\n \t/*\n \t * As rdma-core, UARs are mapped in size of OS page size.\n-\t * Use aligned address to avoid duplicate mmap.\n \t * Ref to libmlx4 function: mlx4_init_context()\n \t */\n \tfor (i = 0; i != txqs_n; ++i) {\n \t\ttxq = dev->data->tx_queues[i];\n \t\tif (!txq)\n \t\t\tcontinue;\n-\t\t/* UAR addr form verbs used to find dup and offset in page. */\n-\t\tuar_va = (uintptr_t)txq->msq.qp_sdb;\n-\t\toff = uar_va & (page_size - 1); /* offset in page. */\n-\t\tuar_va = RTE_ALIGN_FLOOR(uar_va, page_size); /* page addr. */\n-\t\talready_mapped = 0;\n-\t\tfor (j = 0; j != pages_n; ++j) {\n-\t\t\tif (pages[j] == uar_va) {\n-\t\t\t\talready_mapped = 1;\n-\t\t\t\tbreak;\n-\t\t\t}\n-\t\t}\n-\t\t/* new address in reserved UAR address space. */\n-\t\taddr = RTE_PTR_ADD(mlx4_shared_data->uar_base,\n-\t\t\t\t uar_va & (uintptr_t)(MLX4_UAR_SIZE - 1));\n-\t\tif (!already_mapped) {\n-\t\t\tpages[pages_n++] = uar_va;\n-\t\t\t/* fixed mmap to specified address in reserved\n-\t\t\t * address space.\n-\t\t\t */\n-\t\t\tret = mmap(addr, page_size,\n-\t\t\t\t PROT_WRITE, MAP_FIXED | MAP_SHARED, fd,\n-\t\t\t\t txq->msq.uar_mmap_offset);\n-\t\t\tif (ret != addr) {\n-\t\t\t\t/* fixed mmap has to return same address. */\n-\t\t\t\tERROR(\"port %u call to mmap failed on UAR\"\n-\t\t\t\t \" for txq %u\",\n-\t\t\t\t dev->data->port_id, i);\n-\t\t\t\trte_errno = ENXIO;\n-\t\t\t\treturn -rte_errno;\n-\t\t\t}\n+\t\taddr = mmap(NULL, page_size, PROT_WRITE,\n+\t\t\t MAP_FIXED | MAP_SHARED, fd,\n+\t\t\t txq->msq.uar_mmap_offset);\n+\t\tif (addr == MAP_FAILED) {\n+\t\t\tERROR(\"port %u mmap failed for BF reg. of txq %u\",\n+\t\t\t dev->data->port_id, i);\n+\t\t\trte_errno = ENXIO;\n+\t\t\treturn -rte_errno;\n \t\t}\n-\t\tif (rte_eal_process_type() == RTE_PROC_PRIMARY) /* save once. */\n-\t\t\ttxq->msq.db = RTE_PTR_ADD((void *)addr, off);\n-\t\telse\n-\t\t\tassert(txq->msq.db ==\n-\t\t\t RTE_PTR_ADD((void *)addr, off));\n+\t\taddr_ptr = mlx4_uar_get_addr_ptr(txq->bfreg_idx);\n+\t\tif (!addr_ptr)\n+\t\t\treturn -rte_errno;\n+\t\t*addr_ptr = addr;\n \t}\n \treturn 0;\n }\n-#else\n-int\n-mlx4_tx_uar_remap(struct rte_eth_dev *dev __rte_unused, int fd __rte_unused)\n-{\n-\t/*\n-\t * If rdma-core doesn't support UAR remap, secondary process is not\n-\t * supported, thus secondary cannot call this function but only primary\n-\t * makes a call. Return success to not interrupt initialization.\n-\t */\n-\tassert(rte_eal_process_type() == RTE_PROC_PRIMARY);\n-\treturn 0;\n-}\n #endif\n \n /**\n@@ -185,10 +148,8 @@ mlx4_txq_fill_dv_obj_info(struct txq *txq, struct mlx4dv_obj *mlxdv)\n \t\t\t\t (0u << MLX4_SQ_OWNER_BIT));\n #ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET\n \tsq->uar_mmap_offset = dqp->uar_mmap_offset;\n-\tsq->qp_sdb = dqp->sdb;\n-#else\n-\tsq->db = dqp->sdb;\n #endif\n+\t*mlx4_uar_get_addr_ptr(txq->bfreg_idx) = dqp->sdb;\n \tsq->doorbell_qpn = dqp->doorbell_qpn;\n \tcq->buf = dcq->buf.buf;\n \tcq->cqe_cnt = dcq->cqe_cnt;\n@@ -255,6 +216,7 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \tstruct ibv_qp_init_attr qp_init_attr;\n \tstruct txq *txq;\n \tuint8_t *bounce_buf;\n+\tvoid **addr_ptr;\n \tstruct mlx4_malloc_vec vec[] = {\n \t\t{\n \t\t\t.align = RTE_CACHE_LINE_SIZE,\n@@ -429,6 +391,15 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\tgoto error;\n \t}\n #endif\n+\t/* Allocate a new index in UAR table. */\n+\tret = mlx4_uar_alloc_index();\n+\tif (ret < 0)\n+\t\tgoto error;\n+\ttxq->bfreg_idx = ret;\n+\t/* Make sure the local UAR register table is properly expanded. */\n+\taddr_ptr = mlx4_uar_get_addr_ptr(txq->bfreg_idx);\n+\tif (!addr_ptr)\n+\t\tgoto error;\n \tmlx4_txq_fill_dv_obj_info(txq, &mlxdv);\n \t/* Save first wqe pointer in the first element. */\n \t(&(*txq->elts)[0])->wqe =\n@@ -478,6 +449,7 @@ mlx4_tx_queue_release(void *dpdk_txq)\n \t\t\tbreak;\n \t\t}\n \tmlx4_txq_free_elts(txq);\n+\tmlx4_uar_free_index(txq->bfreg_idx);\n \tif (txq->qp)\n \t\tclaim_zero(mlx4_glue->destroy_qp(txq->qp));\n \tif (txq->cq)\n", "prefixes": [ "3/3" ] }