Cover Detail
Show a cover letter.
GET /api/covers/125147/?format=api
{ "id": 125147, "url": "https://patches.dpdk.org/api/covers/125147/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/cover/20230315170342.214127-1-mattias.ronnblom@ericsson.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20230315170342.214127-1-mattias.ronnblom@ericsson.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20230315170342.214127-1-mattias.ronnblom@ericsson.com", "date": "2023-03-15T17:03:40", "name": "[RFC,v2,0/2] Add high-performance timer facility", "submitter": { "id": 1077, "url": "https://patches.dpdk.org/api/people/1077/?format=api", "name": "Mattias Rönnblom", "email": "mattias.ronnblom@ericsson.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/cover/20230315170342.214127-1-mattias.ronnblom@ericsson.com/mbox/", "series": [ { "id": 27404, "url": "https://patches.dpdk.org/api/series/27404/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=27404", "date": "2023-03-15T17:03:40", "name": "Add high-performance timer facility", "version": 2, "mbox": "https://patches.dpdk.org/series/27404/mbox/" } ], "comments": "https://patches.dpdk.org/api/covers/125147/comments/", "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 8D13841EA2;\n\tWed, 15 Mar 2023 18:09:36 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 7169540A7A;\n\tWed, 15 Mar 2023 18:09:36 +0100 (CET)", "from EUR05-AM6-obe.outbound.protection.outlook.com\n (mail-am6eur05on2072.outbound.protection.outlook.com [40.107.22.72])\n by mails.dpdk.org (Postfix) with ESMTP id 91DF240141\n for <dev@dpdk.org>; Wed, 15 Mar 2023 18:09:34 +0100 (CET)", "from AM5PR04CA0035.eurprd04.prod.outlook.com (2603:10a6:206:1::48)\n by DU0PR07MB9625.eurprd07.prod.outlook.com (2603:10a6:10:316::19) with\n Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.6178.26; Wed, 15 Mar\n 2023 17:09:33 +0000", "from AM0EUR02FT018.eop-EUR02.prod.protection.outlook.com\n (2603:10a6:206:1:cafe::b0) by AM5PR04CA0035.outlook.office365.com\n (2603:10a6:206:1::48) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.6178.29 via Frontend\n Transport; Wed, 15 Mar 2023 17:09:33 +0000", "from oa.msg.ericsson.com (192.176.1.74) by\n AM0EUR02FT018.mail.protection.outlook.com (10.13.54.135) with Microsoft SMTP\n Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256) id\n 15.20.6178.20 via Frontend Transport; Wed, 15 Mar 2023 17:09:32 +0000", "from ESESSMB505.ericsson.se (153.88.183.166) by\n ESESBMB502.ericsson.se (153.88.183.169) with Microsoft SMTP Server\n (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256_P256) id\n 15.1.2507.17; Wed, 15 Mar 2023 18:09:32 +0100", "from seliicinfr00050.seli.gic.ericsson.se (153.88.183.153) by\n smtp.internal.ericsson.com (153.88.183.193) with Microsoft SMTP Server id\n 15.1.2507.17 via Frontend Transport; Wed, 15 Mar 2023 18:09:32 +0100", "from breslau.. (seliicwb00002.seli.gic.ericsson.se [10.156.25.100])\n by seliicinfr00050.seli.gic.ericsson.se (Postfix) with ESMTP id\n 8BBAD1C006A; Wed, 15 Mar 2023 18:09:32 +0100 (CET)" ], "ARC-Seal": "i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none;\n b=K3jqrOaIXX7Cn9pP/j0X12SuTtBqOlTB/t+bOnGPggWaAq/BVjVTLaLUYRO9nj9xZTv7I9ptYGHQsBCdSOU9uVXCjqNjRjETU3Ps6qZjU2J7afgobo27KPHi8iaf+hZua4T/XMc1XGJB2jd9CD8mC7HNOSOMjaxM3uMHeDBb9x2TQ6rw4ofFq+bHK8PiQgJAJmjHtWDtKkZkid7npXC4Ea6hx7erR0//TvqIPevDE5+uwxsurcbtHxMpaEdXQdTt0qTC8gvWLCkFD8jcBSfniB3q72z9VYKHVKyRs1T0Dp/tzfYXUMap8HHH72N91WnTfi+uOhhydfWD85taEi+E1w==", "ARC-Message-Signature": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com;\n s=arcselector9901;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1;\n bh=vi4aQsi2xb3OtdvS2MWYT1JpEDuP7N6i6R3m28ZcgAk=;\n b=DGO6M8VNkSEpuJbhLVdF6b/wvI2C+Xp1/7M74ndIorBxv6CD7ppthSiycqIRq940RuWGqh2nFoY/gP16IAHHvaXTB9NQhgOjPJBcTskK9CSx+CWATD2kikV//vZd7ABQUQkLUy7Q2KyZJB34CTS0QwprFhMQ6COpmVwxMpjPt/aO97mktqMYBqo9hSgsfp7C2FTyG7Yi1JjB/bbhDGiePKl2QJHfUyimMh8kwMjmNzDlHAXNSSGV88IktlYzxBWIALWmfUj1f6RuwYBoGxoIcuRTIiTCVYJUX1sBsbpG21PAfXmUfUsOz5v+NJCaPfN7pRADntUFGno2Fh1HtfmNXw==", "ARC-Authentication-Results": "i=1; mx.microsoft.com 1; spf=pass (sender ip is\n 192.176.1.74) smtp.rcpttodomain=dpdk.org smtp.mailfrom=ericsson.com;\n dmarc=pass (p=reject sp=reject pct=100) action=none header.from=ericsson.com;\n dkim=none (message not signed); arc=none", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=ericsson.com;\n s=selector1;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=vi4aQsi2xb3OtdvS2MWYT1JpEDuP7N6i6R3m28ZcgAk=;\n b=TqdH5Ui3zmbBG4B+KducXqPlwIRPrPM+htmEK/1b0D7+nIBwcEd2laUTVACPsho4GJaXb7lG4C6Vw/76wOPYu1ezy0JhlGzybF60qK70o8jJ8chIDwr9+WSJkk0ItK+dlhYMPMvL053979dLhfO6NWQxljnpvR4ezW2Dt9gKSGw=", "X-MS-Exchange-Authentication-Results": "spf=pass (sender IP is 192.176.1.74)\n smtp.mailfrom=ericsson.com; dkim=none (message not signed)\n header.d=none;dmarc=pass action=none header.from=ericsson.com;", "Received-SPF": "Pass (protection.outlook.com: domain of ericsson.com designates\n 192.176.1.74 as permitted sender)\n receiver=protection.outlook.com;\n client-ip=192.176.1.74; helo=oa.msg.ericsson.com; pr=C", "From": "=?utf-8?q?Mattias_R=C3=B6nnblom?= <mattias.ronnblom@ericsson.com>", "To": "<dev@dpdk.org>", "CC": "Erik Gabriel Carrillo <erik.g.carrillo@intel.com>,\n David Marchand <david.marchand@redhat.com>, <maria.lingemark@ericsson.com>,\n Stefan Sundkvist <stefan.sundkvist@ericsson.com>,\n Stephen Hemminger <stephen@networkplumber.org>, =?utf-8?q?Morten_Br=C3=B8ru?=\n\t=?utf-8?q?p?= <mb@smartsharesystems.com>,\n Tyler Retzlaff <roretzla@linux.microsoft.com>, =?utf-8?q?Mattias_R=C3=B6nnb?=\n\t=?utf-8?q?lom?= <mattias.ronnblom@ericsson.com>", "Subject": "[RFC v2 0/2] Add high-performance timer facility", "Date": "Wed, 15 Mar 2023 18:03:40 +0100", "Message-ID": "<20230315170342.214127-1-mattias.ronnblom@ericsson.com>", "X-Mailer": "git-send-email 2.34.1", "In-Reply-To": "<20230228093916.87206-1-mattias.ronnblom@ericsson.com>", "References": "<20230228093916.87206-1-mattias.ronnblom@ericsson.com>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=\"UTF-8\"", "Content-Transfer-Encoding": "8bit", "X-EOPAttributedMessage": "0", "X-MS-PublicTrafficType": "Email", "X-MS-TrafficTypeDiagnostic": "AM0EUR02FT018:EE_|DU0PR07MB9625:EE_", "X-MS-Office365-Filtering-Correlation-Id": "1524b6e9-7c5b-465f-db89-08db25780bf6", "X-MS-Exchange-SenderADCheck": "1", "X-MS-Exchange-AntiSpam-Relay": "0", "X-Microsoft-Antispam": "BCL:0;", "X-Microsoft-Antispam-Message-Info": "\n C/Z465Y+CQS3bfZ0KgecFsKHP1TkQ5HkcFTapEYpvVv4CIbxe/UVsvyFzniOVhLPSaojKhI3P/c1Itk0b3HBB1nkcBjfxMeOoWfbvG9g4+xQzCU7Xieltz8vrvxcLAk1VW1RJenWrI+haWCW9JdgryvfOHDWcdGeduuPgj6hrAm7Oq7gVVwHa9e7/OOasxxQ7rENBvnntlaFWjIg1yOnwNwkM/jiw161fGhF1Q490t3wCV8NmuOM9KKXTbjkhxlJ1BoZvfGvsRMGbH4i2PKO0E0CaDkEL94i0D7vIHtWxAzw42Ng0ONoFdEbLAplJunnVI//wIg0mwUc4QnTm2s/ZA5S8nbE13kOUznvETlp4Qjn1AVXIVPlTm+osSLbcxDuUD1Xa+ESNY0T1pqELs0Di7JTU7pIVVIJh/0gKq5Y4acjmVdWKK2cw6oUAtqVTlM6C6tgxqazkD8UX4I8gNiDsg5uTNZAVtCH3RvmpQ7qvcQYlb7XE62dQI9bG511r1YeQADBu16ZZaAnaoAiRulKoCJqEwuPcVF/38cY1rzON067QQtrMW2QwE14dP05HVTcHoS9WpX4zHkkz/4NqGWCoH1RnBKf6qlq2hJKN0nECtarZgjJzd3aX07YLLsDpKQrk7wV1r43/5I9U+gF1BT4aR9XwPGlr6M0Ld9TFhrOoh9OHT7T7rLuGu5JwJe8i0Da0SDENtW3MgqJgswWFjKRNXVKjJeJaaNyopHKecQPdd7BWI24WojnR3nuJJzOnAss", "X-Forefront-Antispam-Report": "CIP:192.176.1.74; CTRY:SE; LANG:en; SCL:1; SRV:;\n IPV:NLI; SFV:NSPM; H:oa.msg.ericsson.com; PTR:office365.se.ericsson.net;\n CAT:NONE;\n SFS:(13230025)(4636009)(39860400002)(376002)(346002)(396003)(136003)(451199018)(36840700001)(46966006)(40470700004)(1076003)(26005)(41300700001)(36756003)(5660300002)(83380400001)(478600001)(6666004)(107886003)(6266002)(186003)(2616005)(316002)(70206006)(70586007)(8676002)(54906003)(336012)(8936002)(66574015)(40480700001)(4326008)(47076005)(40460700003)(6916009)(7636003)(82740400003)(82960400001)(356005)(82310400005)(86362001)(36860700001)(2906002);\n DIR:OUT; SFP:1101;", "X-OriginatorOrg": "ericsson.com", "X-MS-Exchange-CrossTenant-OriginalArrivalTime": "15 Mar 2023 17:09:32.9462 (UTC)", "X-MS-Exchange-CrossTenant-Network-Message-Id": "\n 1524b6e9-7c5b-465f-db89-08db25780bf6", "X-MS-Exchange-CrossTenant-Id": "92e84ceb-fbfd-47ab-be52-080c6b87953f", "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp": "\n TenantId=92e84ceb-fbfd-47ab-be52-080c6b87953f; Ip=[192.176.1.74];\n Helo=[oa.msg.ericsson.com]", "X-MS-Exchange-CrossTenant-AuthSource": "\n AM0EUR02FT018.eop-EUR02.prod.protection.outlook.com", "X-MS-Exchange-CrossTenant-AuthAs": "Anonymous", "X-MS-Exchange-CrossTenant-FromEntityHeader": "HybridOnPrem", "X-MS-Exchange-Transport-CrossTenantHeadersStamped": "DU0PR07MB9625", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "This patchset is an attempt to introduce a high-performance, highly\nscalable timer facility into DPDK.\n\nMore specifically, the goals for the htimer library are:\n\n* Efficient handling of a handful up to hundreds of thousands of\n concurrent timers.\n* Make adding and canceling timers low-overhead, constant-time\n operations.\n* Provide a service functionally equivalent to that of\n <rte_timer.h>. API/ABI backward compatibility is secondary.\n\nIn the author's opinion, there are two main shortcomings with the\ncurrent DPDK timer library (i.e., rte_timer.[ch]).\n\nOne is the synchronization overhead, where heavy-weight full-barrier\ntype synchronization is used. rte_timer.c uses per-EAL/lcore skip\nlists, but any thread may add or cancel (or otherwise access) timers\nmanaged by another lcore (and thus resides in its timer skip list).\n\nThe other is an algorithmic shortcoming, with rte_timer.c's reliance\non a skip list, which is less efficient than certain alternatives.\n\nThis patchset implements a hierarchical timer wheel (HWT, in\nrte_htw.c), as per the Varghese and Lauck paper \"Hashed and\nHierarchical Timing Wheels: Data Structures for the Efficient\nImplementation of a Timer Facility\". A HWT is a data structure\npurposely design for this task, and used by many operating system\nkernel timer facilities.\n\nTo further improve the solution described by Varghese and Lauck, a\nbitset is placed in front of each of the timer wheel in the HWT,\nreducing overhead of rte_htimer_mgr_manage() (i.e., progressing time\nand expiry processing).\n\nCycle-efficient scanning and manipulation of these bitsets are crucial\nfor the HWT's performance.\n\nThe htimer module keeps a per-lcore (or per-registered EAL thread) HWT\ninstance, much like rte_timer.c keeps a per-lcore skip list.\n\nTo avoid expensive synchronization overhead for thread-local timer\nmanagement, the HWTs are accessed only from the \"owning\" thread. Any\ninteraction any other thread does with a particular lcore's timer\nwheel goes over a set of DPDK rings. A side-effect of this design is\nthat all operations working toward a \"remote\" HWT must be\nasynchronous.\n\nThe <rte_htimer.h> API is available only to EAL threads and registered\nnon-EAL threads.\n\nThe htimer API allows the application to supply the current time,\nuseful in case it already has retrieved this for other purposes,\nsaving the cost of a rdtsc instruction (or its equivalent).\n\nRelative htimer does not retrieve a new time, but reuse the current\ntime (as known via/at-the-time of the manage-call), again to shave off\nsome cycles of overhead.\n\nA semantic improvement compared to the <rte_timer.h> API is that the\nhtimer library can give a definite answer on the question if the timer\nexpiry callback was called, after a timer has been canceled.\n\nThe patchset includes a performance test case\n'timer_htimer_htw_perf_autotest', which compares rte_timer, rte_htimer\nand rte_htw timers in the same scenario.\n\n'timer_htimer_htw_perf_autotest' suggests that rte_htimer is ~3-5x\nfaster than rte_timer for timer/timeout-heavy applications, in a\nscenario where the timer always fires. For a scenario with a mix of\ncanceled and expired timers, the performance difference is greater.\n\nIn scenarios with few timeouts, rte_timer has lower overhead than\nhtimer, but both variants consume very little CPU time.\n\nIn certain scenarios, rte_timer does not suffer from\nnon-constant-time-add and cancel operations. On such is in case the\ntimer added is always last in the list, where htimer is only ~2-3x\nfaster.\n\nThe bitset implementation which the HWT implementation depends upon\nseemed generic-enough and potentially useful outside the world of\nHWTs, to justify being located in the EAL.\n\nThis patchset is very much an RFC, and the author is yet to form an\nopinion on many important issues.\n\n* If deemed a suitable replacement, should the htimer replace the\n current DPDK timer library in some particular (ABI-breaking)\n release, or should it live side-by-side with the then-legacy\n <rte_timer.h> API? A lot of things in and outside DPDK depend on\n <rte_timer.h>, so coexistence may be required to facilitate a smooth\n transition.\n\n* Should the htimer and htw-related files be colocated with rte_timer.c\n in the timer library?\n\n* Would it be useful for applications using asynchronous cancel to\n have the option of having the timer callback run not only in case of\n timer expiration, but also cancellation (on the target lcore)? The\n timer cb signature would need to include an additional parameter in\n that case.\n\n* Should the rte_htimer be a nested struct, so the htw parts be separated\n from the htimer parts?\n\n* <rte_htimer.h> is kept separate from <rte_htimer_mgr.h>, so that\n <rte_htw.h> may avoid a depedency to <rte_htimer_mgr.h>. Should it\n be so?\n\n* rte_htimer struct is only supposed to be used by the application to\n give an indication of how much memory it needs to allocate, and is\n its member are not supposed to be directly accessed (w/ the possible\n exception of the owner_lcore_id field). Should there be a dummy\n struct, or a #define RTE_HTIMER_MEMSIZE or a rte_htimer_get_memsize()\n function instead, serving the same purpose? Better encapsulation,\n but more inconvenient for applications. Run-time dynamic sizing\n would force application-level dynamic allocations.\n\n* Asynchronous cancellation is a little tricky to use for the\n application (primarily due to timer memory reclamation/race\n issues). Should this functionality be removed?\n \n* Should rte_htimer_mgr_init() also retrieve the current time? If so,\n there should to be a variant which allows the user to specify the\n time (to match rte_htimer_mgr_manage_time()). One pitfall with the\n current proposed API is an application calling rte_htimer_mgr_init()\n and then immediately adding a timer with a relative timeout, in\n which case the current absolute time used is 0, which might be a\n surprise.\n\n* Would the event timer adapter be best off using <rte_htw.h>\n directly, or <rte_htimer.h>? In the latter case, there needs to be a\n way to instantiate more HWTs (similar to the \"alt\" functions of\n <rte_timer.h>)?\n\n* Should the PERIODICAL flag (and the complexity it brings) be\n removed? And leave the application with only single-shot timers, and\n the option to re-add them in the timer callback.\n\n* Should the async result codes and the sync cancel error codes be merged\n into one set of result codes?\n\n* Should the rte_htimer_mgr_async_add() have a flag which allow\n buffering add request messages until rte_htimer_mgr_process() is\n called? Or any manage function. Would reduce ring signaling overhead\n (i.e., burst enqueue operations instead of single-element\n enqueue). Could also be a rte_htimer_mgr_async_add_burst() function,\n solving the same \"problem\" a different way. (The signature of such\n a function would not be pretty.)\n\n* Does the functionality provided by the rte_htimer_mgr_process()\n function match its the use cases? Should there me a more clear\n separation between expiry processing and asynchronous operation\n processing?\n\n* Should the patchset be split into more commits? If so, how?\n\nThanks to Erik Carrillo for his assistance.\n\nMattias Rönnblom (2):\n eal: add bitset type\n eal: add high-performance timer facility\n\n app/test/meson.build | 12 +-\n app/test/test_bitset.c | 645 +++++++++++++++++++\n app/test/test_htimer_mgr.c | 674 ++++++++++++++++++++\n app/test/test_htimer_mgr_perf.c | 322 ++++++++++\n app/test/test_htw.c | 478 ++++++++++++++\n app/test/test_htw_perf.c | 181 ++++++\n app/test/test_timer_htimer_htw_perf.c | 693 ++++++++++++++++++++\n doc/api/doxy-api-index.md | 5 +-\n doc/api/doxy-api.conf.in | 1 +\n lib/eal/common/meson.build | 1 +\n lib/eal/common/rte_bitset.c | 29 +\n lib/eal/include/meson.build | 1 +\n lib/eal/include/rte_bitset.h | 879 ++++++++++++++++++++++++++\n lib/eal/version.map | 3 +\n lib/htimer/meson.build | 7 +\n lib/htimer/rte_htimer.h | 68 ++\n lib/htimer/rte_htimer_mgr.c | 547 ++++++++++++++++\n lib/htimer/rte_htimer_mgr.h | 516 +++++++++++++++\n lib/htimer/rte_htimer_msg.h | 44 ++\n lib/htimer/rte_htimer_msg_ring.c | 18 +\n lib/htimer/rte_htimer_msg_ring.h | 55 ++\n lib/htimer/rte_htw.c | 445 +++++++++++++\n lib/htimer/rte_htw.h | 49 ++\n lib/htimer/version.map | 17 +\n lib/meson.build | 1 +\n 25 files changed, 5689 insertions(+), 2 deletions(-)\n create mode 100644 app/test/test_bitset.c\n create mode 100644 app/test/test_htimer_mgr.c\n create mode 100644 app/test/test_htimer_mgr_perf.c\n create mode 100644 app/test/test_htw.c\n create mode 100644 app/test/test_htw_perf.c\n create mode 100644 app/test/test_timer_htimer_htw_perf.c\n create mode 100644 lib/eal/common/rte_bitset.c\n create mode 100644 lib/eal/include/rte_bitset.h\n create mode 100644 lib/htimer/meson.build\n create mode 100644 lib/htimer/rte_htimer.h\n create mode 100644 lib/htimer/rte_htimer_mgr.c\n create mode 100644 lib/htimer/rte_htimer_mgr.h\n create mode 100644 lib/htimer/rte_htimer_msg.h\n create mode 100644 lib/htimer/rte_htimer_msg_ring.c\n create mode 100644 lib/htimer/rte_htimer_msg_ring.h\n create mode 100644 lib/htimer/rte_htw.c\n create mode 100644 lib/htimer/rte_htw.h\n create mode 100644 lib/htimer/version.map" }