Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/64822/?format=api
{ "id": 64822, "url": "https://patches.dpdk.org/api/patches/64822/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20200117042555.22567-1-tyos@jp.ibm.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20200117042555.22567-1-tyos@jp.ibm.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20200117042555.22567-1-tyos@jp.ibm.com", "date": "2020-01-17T04:25:55", "name": "vfio: fix VFIO mapping failures in ppc64le", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "0159929292d3ff00bc38601f9fbcd995431eae27", "submitter": { "id": 1099, "url": "https://patches.dpdk.org/api/people/1099/?format=api", "name": "Takeshi Yoshimura", "email": "tyos@jp.ibm.com" }, "delegate": { "id": 24651, "url": "https://patches.dpdk.org/api/users/24651/?format=api", "username": "dmarchand", "first_name": "David", "last_name": "Marchand", "email": "david.marchand@redhat.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20200117042555.22567-1-tyos@jp.ibm.com/mbox/", "series": [ { "id": 8183, "url": "https://patches.dpdk.org/api/series/8183/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=8183", "date": "2020-01-17T04:25:55", "name": "vfio: fix VFIO mapping failures in ppc64le", "version": 1, "mbox": "https://patches.dpdk.org/series/8183/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/64822/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/64822/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id E412CA051A;\n\tFri, 17 Jan 2020 05:26:24 +0100 (CET)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 7DB881D416;\n\tFri, 17 Jan 2020 05:26:24 +0100 (CET)", "from nh604-vm4.bullet.mail.ssk.yahoo.co.jp\n (nh604-vm4.bullet.mail.ssk.yahoo.co.jp [182.22.90.61])\n by dpdk.org (Postfix) with SMTP id 9899A152A\n for <dev@dpdk.org>; Fri, 17 Jan 2020 05:26:21 +0100 (CET)", "from [182.22.66.106] by nh604.bullet.mail.ssk.yahoo.co.jp with\n NNFMP;\n 17 Jan 2020 04:26:19 -0000", "from [182.22.91.128] by t604.bullet.mail.ssk.yahoo.co.jp with NNFMP;\n 17 Jan 2020 04:26:19 -0000", "from [127.0.0.1] by omp601.mail.ssk.yahoo.co.jp with NNFMP;\n 17 Jan 2020 04:26:19 -0000", "(qmail 90890 invoked by alias); 17 Jan 2020 04:26:19 -0000", "from unknown (HELO localhost.localdomain) (153.226.206.167 with )\n by ymobsmtp5006.mail.kks.ynwp.yahoo.co.jp with SMTP;\n 17 Jan 2020 04:26:19 -0000" ], "X-Yahoo-Newman-Property": "ymail-3", "X-Yahoo-Newman-Id": "832344.56976.bm@omp601.mail.ssk.yahoo.co.jp", "X-YMail-JAS": "\n vrK59e0VM1l87h8uW931fh1CHT5CIEg8g7ITwNB.2UeZsGG9J5mKB6o5_vaMB3AtWXou5wBr4ZYOEpMy81a61XC.eD7iiRkQNqmI_fJBWTZ45v_fAHpn8LZ.6KrFy2ZsmKllgD0nBaanpDo-", "X-Apparently-From": "<endliner2000@yahoo.co.jp>", "X-YMail-OSG": "I04KtRkVM1le1NH9aZaxF_rVXABnDPeULIfgpXenUoxvDZd\n MFqiMd8NVxRqPWT1j6CK6CAikMeYuoq5dbCLQu0cNByei_dH_CqswU81W6.5\n V6PL5bDPQPTj3bhZwfgeEdHHTqSjRUwGzoeTmPYNyyW2d1S_k1edVFd5dVCF\n BsgG_B5Z1dTZG6_DVMhoPkvykAb6IDo_iFKIu59gPmn_kAhinwkQh3SgFBHQ\n VVM6y2tlug0nZ_RfSM2LwwkdPtT4j0w1j45ZfOoZkfFFEINxeew5O4c5N9zS\n JK1Ucuvj.EelDNOpUAWQftKTMyWS1IG6MWYeYuZjwmqLkyWv8feXXpGPcKBv\n dV9t0yCUc5rYheM1e2JDbStDPkbkNNGMqEeTcfDxNRUXEHe71XDa_kh63tgk\n G6J11M65qnyAcg8Zz6pOEMnLx6aFEdAxC3jqYlaf783grudpCy58AYhyALe6\n TUKsdtTVhDT7WH0E0dnxwk7sP.1Y9hh8PTJzLKrCfDtAQ8WJ6MIMnNX9UPrJ\n Kn_q2jxprOwTD4_AWZ393y3m2dlsDcmalVZ127reUk7sv2wPw0NWilcJIXUs\n o83jOHoSU9W0Bj1C59DIYJN3cmSv.n4CGY3g51.opuPXjxPBepjlvIhxbHFj\n FGgpS0dxmGjRshWRtxPHmFJiMlUVH2uDloKC3229IpdGTj3eeiaMrq.6QF50\n IqOIKNPIvGw9DKHIzlcgh2xRWfTInSxb6JDTqxaYtckQjoz7m7u5Oq2HN1BP\n CHV3hVKvCE18-", "From": "Takeshi Yoshimura <tyos@jp.ibm.com>", "To": "dev@dpdk.org", "Cc": "drc@ibm.com,\n\tTakeshi Yoshimura <tyos@jp.ibm.com>", "Date": "Fri, 17 Jan 2020 13:25:55 +0900", "Message-Id": "<20200117042555.22567-1-tyos@jp.ibm.com>", "X-Mailer": "git-send-email 2.20.1 (Apple Git-117)", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[dpdk-dev] [PATCH] vfio: fix VFIO mapping failures in ppc64le", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "ppc64le failed when using large physical memory. I found problems in my two\ncommits in the past.\n\nIn commit e072d16f8920 (\"vfio: fix expanding DMA area in ppc64le\"), I added\na sanity check using a mapped address to resolve an issue around expanding\nIOMMU window, but this was not enough, since memory allocation can return\nmemory anywhere dependent on memory fragmentation. DPDK may still skip DMA\nmapping and attempts to unmap non-mapped DMA during expanding IOMMU window.\nAs a result, SPDK apps using large physical memory frequently failed to\nproceed the communication with NVMe and/or went into an infinite loop.\n\nThe root cause of the bug was in a gap between memory segments managed by\nDPDK and firmware-level DMA mapping. DPDK's memory segments don't contain\nthe state of DMA mapping, and so, the memesg_walk cannot determine if an\niterated memory segment is mapped or not. This resulted in incorrect DMA\nmaps and unmaps.\n\nAt this time, I added the code to avoid iterating non-mapped memory\nsegments during DMA mapping. The memseg_walk iterates over memory segments\nmarked as \"used\", and so, the code sets memory segments that will be\nmapped or unmapped as \"free\" transiently.\n\nThe commit db90b4969e2e (\"vfio: retry creating sPAPR DMA window\") allows\nretring different page levels and sizes to create DMA window. However, this\nallows page sizes different from hugepage sizes. This inconsistency caused\nfailures at the time of DMA mapping after the window creation. This patch\nfixes to retry only different page levels.\n\nFixes: e072d16f8920 (\"vfio: fix expanding DMA area in ppc64le\")\nFixes: db90b4969e2e (\"vfio: retry creating sPAPR DMA window\")\n\nSigned-off-by: Takeshi Yoshimura <tyos@jp.ibm.com>\n---\n lib/librte_eal/linux/eal/eal_vfio.c | 76 +++++++++++++----------------\n 1 file changed, 33 insertions(+), 43 deletions(-)", "diff": "diff --git a/lib/librte_eal/linux/eal/eal_vfio.c b/lib/librte_eal/linux/eal/eal_vfio.c\nindex 95f615c2e..01b5ef3f4 100644\n--- a/lib/librte_eal/linux/eal/eal_vfio.c\n+++ b/lib/librte_eal/linux/eal/eal_vfio.c\n@@ -532,6 +532,17 @@ vfio_mem_event_callback(enum rte_mem_event type, const void *addr, size_t len,\n \t\treturn;\n \t}\n \n+#ifdef RTE_ARCH_PPC_64\n+\tms = rte_mem_virt2memseg(addr, msl);\n+\twhile (cur_len < len) {\n+\t\tint idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);\n+\n+\t\trte_fbarray_set_free(&msl->memseg_arr, idx);\n+\t\tcur_len += ms->len;\n+\t\t++ms;\n+\t}\n+\tcur_len = 0;\n+#endif\n \t/* memsegs are contiguous in memory */\n \tms = rte_mem_virt2memseg(addr, msl);\n \twhile (cur_len < len) {\n@@ -551,6 +562,17 @@ vfio_mem_event_callback(enum rte_mem_event type, const void *addr, size_t len,\n \t\tcur_len += ms->len;\n \t\t++ms;\n \t}\n+#ifdef RTE_ARCH_PPC_64\n+\tcur_len = 0;\n+\tms = rte_mem_virt2memseg(addr, msl);\n+\twhile (cur_len < len) {\n+\t\tint idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);\n+\n+\t\trte_fbarray_set_used(&msl->memseg_arr, idx);\n+\t\tcur_len += ms->len;\n+\t\t++ms;\n+\t}\n+#endif\n }\n \n static int\n@@ -1416,16 +1438,11 @@ vfio_spapr_dma_do_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,\n \treturn 0;\n }\n \n-struct spapr_remap_walk_param {\n-\tint vfio_container_fd;\n-\tuint64_t addr_64;\n-};\n-\n static int\n vfio_spapr_map_walk(const struct rte_memseg_list *msl,\n \t\tconst struct rte_memseg *ms, void *arg)\n {\n-\tstruct spapr_remap_walk_param *param = arg;\n+\tint *vfio_container_fd = arg;\n \n \t/* skip external memory that isn't a heap */\n \tif (msl->external && !msl->heap)\n@@ -1435,10 +1452,7 @@ vfio_spapr_map_walk(const struct rte_memseg_list *msl,\n \tif (ms->iova == RTE_BAD_IOVA)\n \t\treturn 0;\n \n-\tif (ms->addr_64 == param->addr_64)\n-\t\treturn 0;\n-\n-\treturn vfio_spapr_dma_do_map(param->vfio_container_fd, ms->addr_64, ms->iova,\n+\treturn vfio_spapr_dma_do_map(*vfio_container_fd, ms->addr_64, ms->iova,\n \t\t\tms->len, 1);\n }\n \n@@ -1446,7 +1460,7 @@ static int\n vfio_spapr_unmap_walk(const struct rte_memseg_list *msl,\n \t\tconst struct rte_memseg *ms, void *arg)\n {\n-\tstruct spapr_remap_walk_param *param = arg;\n+\tint *vfio_container_fd = arg;\n \n \t/* skip external memory that isn't a heap */\n \tif (msl->external && !msl->heap)\n@@ -1456,17 +1470,13 @@ vfio_spapr_unmap_walk(const struct rte_memseg_list *msl,\n \tif (ms->iova == RTE_BAD_IOVA)\n \t\treturn 0;\n \n-\tif (ms->addr_64 == param->addr_64)\n-\t\treturn 0;\n-\n-\treturn vfio_spapr_dma_do_map(param->vfio_container_fd, ms->addr_64, ms->iova,\n+\treturn vfio_spapr_dma_do_map(*vfio_container_fd, ms->addr_64, ms->iova,\n \t\t\tms->len, 0);\n }\n \n struct spapr_walk_param {\n \tuint64_t window_size;\n \tuint64_t hugepage_sz;\n-\tuint64_t addr_64;\n };\n \n static int\n@@ -1484,10 +1494,6 @@ vfio_spapr_window_size_walk(const struct rte_memseg_list *msl,\n \tif (ms->iova == RTE_BAD_IOVA)\n \t\treturn 0;\n \n-\t/* do not iterate ms we haven't mapped yet */\n-\tif (param->addr_64 && ms->addr_64 == param->addr_64)\n-\t\treturn 0;\n-\n \tif (max > param->window_size) {\n \t\tparam->hugepage_sz = ms->hugepage_sz;\n \t\tparam->window_size = max;\n@@ -1531,20 +1537,11 @@ vfio_spapr_create_new_dma_window(int vfio_container_fd,\n \t\t/* try possible page_shift and levels for workaround */\n \t\tuint32_t levels;\n \n-\t\tfor (levels = 1; levels <= info.ddw.levels; levels++) {\n-\t\t\tuint32_t pgsizes = info.ddw.pgsizes;\n-\n-\t\t\twhile (pgsizes != 0) {\n-\t\t\t\tcreate->page_shift = 31 - __builtin_clz(pgsizes);\n-\t\t\t\tcreate->levels = levels;\n-\t\t\t\tret = ioctl(vfio_container_fd,\n-\t\t\t\t\tVFIO_IOMMU_SPAPR_TCE_CREATE, create);\n-\t\t\t\tif (!ret)\n-\t\t\t\t\tbreak;\n-\t\t\t\tpgsizes &= ~(1 << create->page_shift);\n-\t\t\t}\n-\t\t\tif (!ret)\n-\t\t\t\tbreak;\n+\t\tfor (levels = create->levels + 1;\n+\t\t\tret && levels <= info.ddw.levels; levels++) {\n+\t\t\tcreate->levels = levels;\n+\t\t\tret = ioctl(vfio_container_fd,\n+\t\t\t\tVFIO_IOMMU_SPAPR_TCE_CREATE, create);\n \t\t}\n #endif\n \t\tif (ret) {\n@@ -1585,7 +1582,6 @@ vfio_spapr_dma_mem_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,\n \n \t/* check if window size needs to be adjusted */\n \tmemset(¶m, 0, sizeof(param));\n-\tparam.addr_64 = vaddr;\n \n \t/* we're inside a callback so use thread-unsafe version */\n \tif (rte_memseg_walk_thread_unsafe(vfio_spapr_window_size_walk,\n@@ -1610,14 +1606,9 @@ vfio_spapr_dma_mem_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,\n \tif (do_map) {\n \t\t/* re-create window and remap the entire memory */\n \t\tif (iova + len > create.window_size) {\n-\t\t\tstruct spapr_remap_walk_param remap_param = {\n-\t\t\t\t.vfio_container_fd = vfio_container_fd,\n-\t\t\t\t.addr_64 = vaddr,\n-\t\t\t};\n-\n \t\t\t/* release all maps before recreating the window */\n \t\t\tif (rte_memseg_walk_thread_unsafe(vfio_spapr_unmap_walk,\n-\t\t\t\t\t&remap_param) < 0) {\n+\t\t\t\t\t&vfio_container_fd) < 0) {\n \t\t\t\tRTE_LOG(ERR, EAL, \"Could not release DMA maps\\n\");\n \t\t\t\tret = -1;\n \t\t\t\tgoto out;\n@@ -1644,7 +1635,7 @@ vfio_spapr_dma_mem_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,\n \t\t\t/* we're inside a callback, so use thread-unsafe version\n \t\t\t */\n \t\t\tif (rte_memseg_walk_thread_unsafe(vfio_spapr_map_walk,\n-\t\t\t\t\t&remap_param) < 0) {\n+\t\t\t\t\t&vfio_container_fd) < 0) {\n \t\t\t\tRTE_LOG(ERR, EAL, \"Could not recreate DMA maps\\n\");\n \t\t\t\tret = -1;\n \t\t\t\tgoto out;\n@@ -1691,7 +1682,6 @@ vfio_spapr_dma_map(int vfio_container_fd)\n \tstruct spapr_walk_param param;\n \n \tmemset(¶m, 0, sizeof(param));\n-\tparam.addr_64 = 0UL;\n \n \t/* create DMA window from 0 to max(phys_addr + len) */\n \trte_memseg_walk(vfio_spapr_window_size_walk, ¶m);\n", "prefixes": [] }