Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/23406/?format=api
{ "id": 23406, "url": "http://patches.dpdk.org/api/patches/23406/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1491811459-1647-1-git-send-email-i.maximets@samsung.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1491811459-1647-1-git-send-email-i.maximets@samsung.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1491811459-1647-1-git-send-email-i.maximets@samsung.com", "date": "2017-04-10T08:04:19", "name": "[dpdk-dev,v2] mem: balanced allocation of hugepages", "commit_ref": null, "pull_url": null, "state": "changes-requested", "archived": true, "hash": "aac950023a90f9b6c2ec8fa3ff0f9120f62beed8", "submitter": { "id": 323, "url": "http://patches.dpdk.org/api/people/323/?format=api", "name": "Ilya Maximets", "email": "i.maximets@samsung.com" }, "delegate": null, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1491811459-1647-1-git-send-email-i.maximets@samsung.com/mbox/", "series": [], "comments": "http://patches.dpdk.org/api/patches/23406/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/23406/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [IPv6:::1])\n\tby dpdk.org (Postfix) with ESMTP id 3FCA029C7;\n\tMon, 10 Apr 2017 10:04:30 +0200 (CEST)", "from mailout2.w1.samsung.com (mailout2.w1.samsung.com\n\t[210.118.77.12]) by dpdk.org (Postfix) with ESMTP id 976362C5\n\tfor <dev@dpdk.org>; Mon, 10 Apr 2017 10:04:28 +0200 (CEST)", "from eucas1p1.samsung.com (unknown [182.198.249.206])\n\tby mailout2.w1.samsung.com\n\t(Oracle Communications Messaging Server 7.0.5.31.0 64bit (built May 5\n\t2014)) with ESMTP id <0OO600BA4P3EYK60@mailout2.w1.samsung.com> for\n\tdev@dpdk.org; Mon, 10 Apr 2017 09:04:26 +0100 (BST)", "from eusmges2.samsung.com (unknown [203.254.199.241])\n\tby\teucas1p1.samsung.com (KnoxPortal) with ESMTP\n\tid\t20170410080426eucas1p1003a5d470c270861a4ea3d2df369e70f~z_p-k80hM0277902779eucas1p1y;\n\tMon, 10 Apr 2017 08:04:26 +0000 (GMT)", "from eucas1p2.samsung.com ( [182.198.249.207])\n\tby\teusmges2.samsung.com (EUCPMTA) with SMTP id ED.1F.04459.98C3BE85;\n\tMon, 10\tApr 2017 09:04:25 +0100 (BST)", "from eusmgms1.samsung.com (unknown [182.198.249.179])\n\tby\teucas1p2.samsung.com (KnoxPortal) with ESMTP\n\tid\t20170410080425eucas1p27fd424ae58151f13b1a7a3723aa4ad1e~z_p_8_DLT0505105051eucas1p2b;\n\tMon, 10 Apr 2017 08:04:25 +0000 (GMT)", "from eusync1.samsung.com ( [203.254.199.211])\n\tby\teusmgms1.samsung.com (EUCPMTA) with SMTP id 62.11.17452.30D3BE85;\n\tMon, 10\tApr 2017 09:06:27 +0100 (BST)", "from imaximets.rnd.samsung.ru ([106.109.129.180])\n\tby\teusync1.samsung.com\n\t(Oracle Communications Messaging Server 7.0.5.31.0 64bit\t(built May 5\n\t2014)) with ESMTPA id <0OO6004ZRP39JL10@eusync1.samsung.com>;\tMon,\n\t10 Apr 2017 09:04:25 +0100 (BST)" ], "X-AuditID": "cbfec7f1-f796e6d00000116b-e9-58eb3c896ffd", "From": "Ilya Maximets <i.maximets@samsung.com>", "To": "dev@dpdk.org, David Marchand <david.marchand@6wind.com>,\n\tSergio Gonzalez Monroy <sergio.gonzalez.monroy@intel.com>,\n\tThomas Monjalon <thomas.monjalon@6wind.com>", "Cc": "Heetae Ahn <heetae82.ahn@samsung.com>,\n\tYuanhan Liu <yuanhan.liu@linux.intel.com>,\n\tJianfeng Tan <jianfeng.tan@intel.com>,\n\tNeil Horman <nhorman@tuxdriver.com>, Yulong Pei <yulong.pei@intel.com>,\n\tIlya Maximets <i.maximets@samsung.com>", "Date": "Mon, 10 Apr 2017 11:04:19 +0300", "Message-id": "<1491811459-1647-1-git-send-email-i.maximets@samsung.com>", "X-Mailer": "git-send-email 2.7.4", "In-reply-to": "<1487250070-13973-1-git-send-email-i.maximets@samsung.com>", "X-Brightmail-Tracker": [ "H4sIAAAAAAAAA+NgFrrHIsWRmVeSWpSXmKPExsWy7djP87qdNq8jDKY+trZY0dHOYvHu03Ym\n\ti2mfb7NbXGn/yW7RPfsLm8Wt5pNsFismHGG0+LJpOpvF9QkXWC2+PfjO7MDlcbH/DqPHrwVL\n\tWT0W73nJ5DHvZKBH35ZVjB5Xvq9mDGCL4rJJSc3JLEst0rdL4Mr4vv0wW8Fjw4qzl+8wNTD+\n\tUe9i5OSQEDCR6J4zixHCFpO4cG89WxcjF4eQwFJGiR3rFzFDOJ8ZJX49/8oG0/Hw3h1WEFtI\n\tYBmjxOsPahBFzUwSG/5CFLEJ6EicWn2EESQhIrCcUWL9l3tgO5gFPjJKtPz262Lk4BAWsJS4\n\t+S8GJMwioCqxqfkHC4jNK+Aq0fLkL9RJchI3z3Uyg9icAu4SjyZvYQeZKSEwmV1i+c7rjCBz\n\tJARkJTYdYIaod5FoO/IXyhaWeHUcpB7ElpHo7DjIBNHbzCjRsOoSI4QzgVHiS/NyJogqe4lT\n\tN68yQRzKJzFp23RmiAW8Eh1tQhAlHhL7D31lhbAdJd4s3cMO8f0sYHgtf84+gVFmASPDKkaR\n\t1NLi3PTUYiO94sTc4tK8dL3k/NxNjMBYP/3v+McdjO9PWB1iFOBgVOLh/VHxKkKINbGsuDL3\n\tEKMEB7OSCO8N6dcRQrwpiZVVqUX58UWlOanFhxilOViUxHm5Tl2LEBJITyxJzU5NLUgtgsky\n\tcXBKNTC6dbnm3D3otFqWN3D/uxjZpF6faxv9px4o31p3TWLShVfTDnsv2b9g5c6Ptxc9iz64\n\t603DKv7OiogAjeVFCfHmS9+7WXLxRyXX++6xtRNk5J7hsjyObZfzvyvbEiex7365QEMmU7VM\n\t5doB67dMMYsX/ErSnXpxu9D6sMzfaRcMvIuWd9xIKjupxFKckWioxVxUnAgAQ71mNvECAAA=", "H4sIAAAAAAAAA+NgFvrHLMWRmVeSWpSXmKPExsVy+t/xy7rMtq8jDJomKFus6GhnsXj3aTuT\n\txbTPt9ktrrT/ZLfonv2FzeJW80k2ixUTjjBafNk0nc3i+oQLrBbfHnxnduDyuNh/h9Hj14Kl\n\trB6L97xk8ph3MtCjb8sqRo8r31czBrBFudlkpCampBYppOYl56dk5qXbKoWGuOlaKCnkJeam\n\t2ipF6PqGBCkplCXmlAJ5RgZowME5wD1YSd8uwS3j+/bDbAWPDSvOXr7D1MD4R72LkZNDQsBE\n\t4uG9O6wQtpjEhXvr2boYuTiEBJYwSkz88QrKaWWSaF//hR2kik1AR+LU6iOMIAkRgeWMEos6\n\t+sCqmAU+Mkoc3/GHpYuRg0NYwFLi5r8YkAYWAVWJTc0/WEBsXgFXiZYnfxkh1slJ3DzXyQxi\n\tcwq4SzyavAVsgZCAm0TXoocsExh5FzAyrGIUSS0tzk3PLTbUK07MLS7NS9dLzs/dxAgM+23H\n\tfm7ewXhpY/AhRgEORiUe3oDqVxFCrIllxZW5hxglOJiVRHhvSL+OEOJNSaysSi3Kjy8qzUkt\n\tPsRoCnTURGYp0eR8YEzmlcQbmhiaWxoaGVtYmBsZKYnzlny4Ei4kkJ5YkpqdmlqQWgTTx8TB\n\tKdXA6Olb05Mq/ee+gF2nocUX5pddf9SnmCy+V/q8dIGY5MInx7dYNi5dpb9z858dZxfsNHxZ\n\tvmbZtVNKFdKKYo1ROaY9R/cmLwxyzHBwNfDWOSM9pWE2r2SloW7qQ9Okp2kR9z4J6i3tdT0h\n\tuFK0//KeBRrPTBia8rWWTXb9I/D+5r0Pdziq82d+V2Ipzkg01GIuKk4EAIyTKcmRAgAA" ], "X-MTR": "20000000000000000@CPGS", "X-CMS-MailID": "20170410080425eucas1p27fd424ae58151f13b1a7a3723aa4ad1e", "X-Msg-Generator": "CA", "X-Sender-IP": "182.198.249.179", "X-Local-Sender": "=?utf-8?q?Ilya_Maximets=1BSRR-Virtualization_Lab=1B?=\n\t=?utf-8?b?7IK87ISx7KCE7J6QG0xlYWRpbmcgRW5naW5lZXI=?=", "X-Global-Sender": "=?utf-8?q?Ilya_Maximets=1BSRR-Virtualization_Lab=1BSamsu?=\n\t=?utf-8?q?ng_Electronics=1BLeading_Engineer?=", "X-Sender-Code": "=?utf-8?q?C10=1BCISHQ=1BC10GD01GD010154?=", "CMS-TYPE": "201P", "X-HopCount": "7", "X-CMS-RootMailID": "20170410080425eucas1p27fd424ae58151f13b1a7a3723aa4ad1e", "X-RootMTR": "20170410080425eucas1p27fd424ae58151f13b1a7a3723aa4ad1e", "References": "<1487250070-13973-1-git-send-email-i.maximets@samsung.com>\n\t<CGME20170410080425eucas1p27fd424ae58151f13b1a7a3723aa4ad1e@eucas1p2.samsung.com>", "Subject": "[dpdk-dev] [PATCH v2] mem: balanced allocation of hugepages", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<http://dpdk.org/ml/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://dpdk.org/ml/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<http://dpdk.org/ml/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Currently EAL allocates hugepages one by one not paying\nattention from which NUMA node allocation was done.\n\nSuch behaviour leads to allocation failure if number of\navailable hugepages for application limited by cgroups\nor hugetlbfs and memory requested not only from the first\nsocket.\n\nExample:\n\t# 90 x 1GB hugepages availavle in a system\n\n\tcgcreate -g hugetlb:/test\n\t# Limit to 32GB of hugepages\n\tcgset -r hugetlb.1GB.limit_in_bytes=34359738368 test\n\t# Request 4GB from each of 2 sockets\n\tcgexec -g hugetlb:test testpmd --socket-mem=4096,4096 ...\n\n\tEAL: SIGBUS: Cannot mmap more hugepages of size 1024 MB\n\tEAL: 32 not 90 hugepages of size 1024 MB allocated\n\tEAL: Not enough memory available on socket 1!\n\t Requested: 4096MB, available: 0MB\n\tPANIC in rte_eal_init():\n\tCannot init memory\n\n\tThis happens beacause all allocated pages are\n\ton socket 0.\n\nFix this issue by setting mempolicy MPOL_PREFERRED for each\nhugepage to one of requested nodes in a round-robin fashion.\nIn this case all allocated pages will be fairly distributed\nbetween all requested nodes.\n\nNew config option RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES\nintroduced and disabled by default because of external\ndependency from libnuma.\n\nFixes: 77988fc08dc5 (\"mem: fix allocating all free hugepages\")\n\nSigned-off-by: Ilya Maximets <i.maximets@samsung.com>\n---\n\nVersion 2:\n\t* rebased (fuzz in Makefile)\n\n config/common_base | 1 +\n lib/librte_eal/Makefile | 4 ++\n lib/librte_eal/linuxapp/eal/eal_memory.c | 65 ++++++++++++++++++++++++++++++++\n mk/rte.app.mk | 3 ++\n 4 files changed, 73 insertions(+)", "diff": "diff --git a/config/common_base b/config/common_base\nindex 5f2ad94..09782ff 100644\n--- a/config/common_base\n+++ b/config/common_base\n@@ -102,6 +102,7 @@ CONFIG_RTE_EAL_ALWAYS_PANIC_ON_ERROR=n\n CONFIG_RTE_EAL_IGB_UIO=n\n CONFIG_RTE_EAL_VFIO=n\n CONFIG_RTE_MALLOC_DEBUG=n\n+CONFIG_RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES=n\n \n # Default driver path (or \"\" to disable)\n CONFIG_RTE_EAL_PMD_PATH=\"\"\ndiff --git a/lib/librte_eal/Makefile b/lib/librte_eal/Makefile\nindex 5690bb4..e5f552a 100644\n--- a/lib/librte_eal/Makefile\n+++ b/lib/librte_eal/Makefile\n@@ -37,4 +37,8 @@ DEPDIRS-linuxapp := common\n DIRS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += bsdapp\n DEPDIRS-bsdapp := common\n \n+ifeq ($(CONFIG_RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES),y)\n+LDLIBS += -lnuma\n+endif\n+\n include $(RTE_SDK)/mk/rte.subdir.mk\ndiff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c\nindex 657c6f4..8cb7432 100644\n--- a/lib/librte_eal/linuxapp/eal/eal_memory.c\n+++ b/lib/librte_eal/linuxapp/eal/eal_memory.c\n@@ -83,6 +83,9 @@\n #include <sys/time.h>\n #include <signal.h>\n #include <setjmp.h>\n+#ifdef RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES\n+#include <numaif.h>\n+#endif\n \n #include <rte_log.h>\n #include <rte_memory.h>\n@@ -377,6 +380,21 @@ static int huge_wrap_sigsetjmp(void)\n \treturn sigsetjmp(huge_jmpenv, 1);\n }\n \n+#ifdef RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES\n+#ifndef ULONG_SIZE\n+#define ULONG_SIZE sizeof(unsigned long)\n+#endif\n+#ifndef ULONG_BITS\n+#define ULONG_BITS (ULONG_SIZE * CHAR_BIT)\n+#endif\n+#ifndef DIV_ROUND_UP\n+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))\n+#endif\n+#ifndef BITS_TO_LONGS\n+#define BITS_TO_LONGS(nr) DIV_ROUND_UP(nr, ULONG_SIZE)\n+#endif\n+#endif\n+\n /*\n * Mmap all hugepages of hugepage table: it first open a file in\n * hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the\n@@ -393,10 +411,48 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl,\n \tvoid *virtaddr;\n \tvoid *vma_addr = NULL;\n \tsize_t vma_len = 0;\n+#ifdef RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES\n+\tunsigned long nodemask[BITS_TO_LONGS(RTE_MAX_NUMA_NODES)] = {0UL};\n+\tunsigned long maxnode = 0;\n+\tint node_id = -1;\n+\n+\tfor (i = 0; i < RTE_MAX_NUMA_NODES; i++)\n+\t\tif (internal_config.socket_mem[i])\n+\t\t\tmaxnode = i + 1;\n+#endif\n \n \tfor (i = 0; i < hpi->num_pages[0]; i++) {\n \t\tuint64_t hugepage_sz = hpi->hugepage_sz;\n \n+#ifdef RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES\n+\t\tif (maxnode) {\n+\t\t\tnode_id = (node_id + 1) % RTE_MAX_NUMA_NODES;\n+\t\t\twhile (!internal_config.socket_mem[node_id])\n+\t\t\t\tnode_id = (node_id + 1) % RTE_MAX_NUMA_NODES;\n+\n+\t\t\tnodemask[node_id / ULONG_BITS] =\n+\t\t\t\t\t\t1UL << (node_id % ULONG_BITS);\n+\n+\t\t\tRTE_LOG(DEBUG, EAL,\n+\t\t\t\t\"Setting policy MPOL_PREFERRED for socket %d\\n\",\n+\t\t\t\tnode_id);\n+\t\t\t/*\n+\t\t\t * Due to old linux kernel bug (feature?) we have to\n+\t\t\t * increase maxnode by 1. It will be unconditionally\n+\t\t\t * decreased back to normal value inside the syscall\n+\t\t\t * handler.\n+\t\t\t */\n+\t\t\tif (set_mempolicy(MPOL_PREFERRED,\n+\t\t\t\t\t nodemask, maxnode + 1) < 0) {\n+\t\t\t\tRTE_LOG(ERR, EAL,\n+\t\t\t\t\t\"Failed to set policy MPOL_PREFERRED: \"\n+\t\t\t\t\t\"%s\\n\", strerror(errno));\n+\t\t\t\treturn i;\n+\t\t\t}\n+\n+\t\t\tnodemask[node_id / ULONG_BITS] = 0UL;\n+\t\t}\n+#endif\n \t\tif (orig) {\n \t\t\thugepg_tbl[i].file_id = i;\n \t\t\thugepg_tbl[i].size = hugepage_sz;\n@@ -507,6 +563,10 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl,\n \t\tvma_len -= hugepage_sz;\n \t}\n \n+#ifdef RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES\n+\tif (maxnode && set_mempolicy(MPOL_DEFAULT, NULL, 0) < 0)\n+\t\tRTE_LOG(ERR, EAL, \"Failed to set mempolicy MPOL_DEFAULT\\n\");\n+#endif\n \treturn i;\n }\n \n@@ -591,6 +651,11 @@ find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)\n \t\t\tif (hugepg_tbl[i].orig_va == va) {\n \t\t\t\thugepg_tbl[i].socket_id = socket_id;\n \t\t\t\thp_count++;\n+#ifdef RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES\n+\t\t\t\tRTE_LOG(DEBUG, EAL,\n+\t\t\t\t\t\"Hugepage %s is on socket %d\\n\",\n+\t\t\t\t\thugepg_tbl[i].filepath, socket_id);\n+#endif\n \t\t\t}\n \t\t}\n \t}\ndiff --git a/mk/rte.app.mk b/mk/rte.app.mk\nindex 4c659e9..ca8e5fe 100644\n--- a/mk/rte.app.mk\n+++ b/mk/rte.app.mk\n@@ -173,6 +173,9 @@ ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),n)\n # The static libraries do not know their dependencies.\n # So linking with static library requires explicit dependencies.\n _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrt\n+ifeq ($(CONFIG_RTE_LIBRTE_EAL_NUMA_AWARE_HUGEPAGES),y)\n+_LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lnuma\n+endif\n _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lm\n _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrt\n _LDLIBS-$(CONFIG_RTE_LIBRTE_METER) += -lm\n", "prefixes": [ "dpdk-dev", "v2" ] }