From patchwork Tue Oct 1 18:28:55 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Honnappa Nagarahalli X-Patchwork-Id: 60362 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 17C1BCFA6; Tue, 1 Oct 2019 20:29:16 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by dpdk.org (Postfix) with ESMTP id E044191 for ; Tue, 1 Oct 2019 20:29:12 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0C3F01570; Tue, 1 Oct 2019 11:29:12 -0700 (PDT) Received: from qc2400f-1.austin.arm.com (qc2400f-1.austin.arm.com [10.118.12.34]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id EA6403F918; Tue, 1 Oct 2019 11:29:11 -0700 (PDT) From: Honnappa Nagarahalli To: bruce.richardson@intel.com, vladimir.medvedkin@intel.com, olivier.matz@6wind.com Cc: dev@dpdk.org, konstantin.ananyev@intel.com, stephen@networkplumber.org, paulmck@linux.ibm.com, Gavin.Hu@arm.com, Honnappa.Nagarahalli@arm.com, Dharmik.Thakkar@arm.com, Ruifeng.Wang@arm.com, nd@arm.com, Ruifeng Wang Date: Tue, 1 Oct 2019 13:28:55 -0500 Message-Id: <20191001182857.43867-2-honnappa.nagarahalli@arm.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20191001182857.43867-1-honnappa.nagarahalli@arm.com> References: <20190906094534.36060-1-ruifeng.wang@arm.com> <20191001182857.43867-1-honnappa.nagarahalli@arm.com> Subject: [dpdk-dev] [PATCH v3 1/3] lib/lpm: integrate RCU QSBR X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Ruifeng Wang Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results. RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries. Signed-off-by: Ruifeng Wang Reviewed-by: Honnappa Nagarahalli --- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++++++++++++++++++++++---- lib/librte_lpm/rte_lpm.h | 21 ++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 5 files changed, 122 insertions(+), 12 deletions(-) diff --git a/lib/librte_lpm/Makefile b/lib/librte_lpm/Makefile index a7946a1c5..ca9e16312 100644 --- a/lib/librte_lpm/Makefile +++ b/lib/librte_lpm/Makefile @@ -6,9 +6,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # library name LIB = librte_lpm.a +CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.map diff --git a/lib/librte_lpm/meson.build b/lib/librte_lpm/meson.build index a5176d8ae..19a35107f 100644 --- a/lib/librte_lpm/meson.build +++ b/lib/librte_lpm/meson.build @@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 2 +allow_experimental_apis = true sources = files('rte_lpm.c', 'rte_lpm6.c') headers = files('rte_lpm.h', 'rte_lpm6.h') # since header files have different names, we can install all vector headers # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu'] diff --git a/lib/librte_lpm/rte_lpm.c b/lib/librte_lpm/rte_lpm.c index 3a929a1b1..ca58d4b35 100644 --- a/lib/librte_lpm/rte_lpm.c +++ b/lib/librte_lpm/rte_lpm.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2019 Arm Limited */ #include @@ -381,6 +382,8 @@ rte_lpm_free_v1604(struct rte_lpm *lpm) rte_mcfg_tailq_write_unlock(); + if (lpm->dq) + rte_rcu_qsbr_dq_delete(lpm->dq); rte_free(lpm->tbl8); rte_free(lpm->rules_tbl); rte_free(lpm); @@ -390,6 +393,59 @@ BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); MAP_STATIC_SYMBOL(void rte_lpm_free(struct rte_lpm *lpm), rte_lpm_free_v1604); +struct __rte_lpm_rcu_dq_entry { + uint32_t tbl8_group_index; + uint32_t pad; +}; + +static void +__lpm_rcu_qsbr_free_resource(void *p, void *data) +{ + struct rte_lpm_tbl_entry zero_tbl8_entry = {0}; + struct __rte_lpm_rcu_dq_entry *e = + (struct __rte_lpm_rcu_dq_entry *)data; + struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + + /* Set tbl8 group invalid */ + __atomic_store(&tbl8[e->tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. + */ +int +rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr *v) +{ + char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE]; + struct rte_rcu_qsbr_dq_parameters params; + + if ((lpm == NULL) || (v == NULL)) { + rte_errno = EINVAL; + return 1; + } + + if (lpm->dq) { + rte_errno = EEXIST; + return 1; + } + + /* Init QSBR defer queue. */ + snprintf(rcu_dq_name, sizeof(rcu_dq_name), "LPM_RCU_%s", lpm->name); + params.name = rcu_dq_name; + params.size = lpm->number_tbl8s; + params.esize = sizeof(struct __rte_lpm_rcu_dq_entry); + params.f = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8; + params.v = v; + lpm->dq = rte_rcu_qsbr_dq_create(¶ms); + if (lpm->dq == NULL) { + RTE_LOG(ERR, LPM, "LPM QS defer queue creation failed\n"); + return 1; + } + + return 0; +} + /* * Adds a rule to the rule table. * @@ -679,14 +735,15 @@ tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8) } static int32_t -tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) +__tbl8_alloc_v1604(struct rte_lpm *lpm) { uint32_t group_idx; /* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry; /* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ - for (group_idx = 0; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx * RTE_LPM_TBL8_GROUP_NUM_ENTRIES]; + for (group_idx = 0; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * + RTE_LPM_TBL8_GROUP_NUM_ENTRIES]; /* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) { struct rte_lpm_tbl_entry new_tbl8_entry = { @@ -712,6 +769,21 @@ tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) return -ENOSPC; } +static int32_t +tbl8_alloc_v1604(struct rte_lpm *lpm) +{ + int32_t group_idx; /* tbl8 group index. */ + + group_idx = __tbl8_alloc_v1604(lpm); + if ((group_idx < 0) && (lpm->dq != NULL)) { + /* If there are no tbl8 groups try to reclaim some. */ + if (rte_rcu_qsbr_dq_reclaim(lpm->dq) == 0) + group_idx = __tbl8_alloc_v1604(lpm); + } + + return group_idx; +} + static void tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start) { @@ -728,13 +800,21 @@ tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start) } static void -tbl8_free_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t tbl8_group_start) +tbl8_free_v1604(struct rte_lpm *lpm, uint32_t tbl8_group_start) { - /* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0}; + struct __rte_lpm_rcu_dq_entry e; - __atomic_store(&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); + if (lpm->dq != NULL) { + e.tbl8_group_index = tbl8_group_start; + e.pad = 0; + /* Push into QSBR defer queue. */ + rte_rcu_qsbr_dq_enqueue(lpm->dq, (void *)&e); + } else { + /* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start], &zero_tbl8_entry, + __ATOMIC_RELAXED); + } } static __rte_noinline int32_t @@ -1037,7 +1117,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, if (!lpm->tbl24[tbl24_index].valid) { /* Search for a free tbl8 group. */ - tbl8_group_index = tbl8_alloc_v1604(lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index = tbl8_alloc_v1604(lpm); /* Check tbl8 allocation was successful. */ if (tbl8_group_index < 0) { @@ -1083,7 +1163,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, } /* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group == 0) { /* Search for free tbl8 group. */ - tbl8_group_index = tbl8_alloc_v1604(lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index = tbl8_alloc_v1604(lpm); if (tbl8_group_index < 0) { return tbl8_group_index; @@ -1818,7 +1898,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, */ lpm->tbl24[tbl24_index].valid = 0; __atomic_thread_fence(__ATOMIC_RELEASE); - tbl8_free_v1604(lpm->tbl8, tbl8_group_start); + tbl8_free_v1604(lpm, tbl8_group_start); } else if (tbl8_recycle_index > -1) { /* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = { @@ -1834,7 +1914,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, __atomic_store(&lpm->tbl24[tbl24_index], &new_tbl24_entry, __ATOMIC_RELAXED); __atomic_thread_fence(__ATOMIC_RELEASE); - tbl8_free_v1604(lpm->tbl8, tbl8_group_start); + tbl8_free_v1604(lpm, tbl8_group_start); } #undef group_idx return 0; diff --git a/lib/librte_lpm/rte_lpm.h b/lib/librte_lpm/rte_lpm.h index 906ec4483..49c12a68d 100644 --- a/lib/librte_lpm/rte_lpm.h +++ b/lib/librte_lpm/rte_lpm.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2019 Arm Limited */ #ifndef _RTE_LPM_H_ @@ -21,6 +22,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -186,6 +188,7 @@ struct rte_lpm { __rte_cache_aligned; /**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8; /**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl; /**< LPM rules. */ + struct rte_rcu_qsbr_dq *dq; /**< RCU QSBR defer queue.*/ }; /** @@ -248,6 +251,24 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm); void rte_lpm_free_v1604(struct rte_lpm *lpm); +/** + * Associate RCU QSBR variable with an LPM object. + * + * @param lpm + * the lpm object to add RCU QSBR + * @param v + * RCU QSBR variable + * @return + * On success - 0 + * On error - 1 with error code set in rte_errno. + * Possible rte_errno codes are: + * - EINVAL - invalid pointer + * - EEXIST - already added QSBR + * - ENOMEM - memory allocation failure + */ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr *v); + /** * Add a rule to the LPM table. * diff --git a/lib/librte_lpm/rte_lpm_version.map b/lib/librte_lpm/rte_lpm_version.map index 90beac853..b353aabd2 100644 --- a/lib/librte_lpm/rte_lpm_version.map +++ b/lib/librte_lpm/rte_lpm_version.map @@ -44,3 +44,9 @@ DPDK_17.05 { rte_lpm6_lookup_bulk_func; } DPDK_16.04; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +}; From patchwork Tue Oct 1 18:28:56 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Honnappa Nagarahalli X-Patchwork-Id: 60363 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 5AA321BE89; Tue, 1 Oct 2019 20:29:18 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by dpdk.org (Postfix) with ESMTP id E8AF94C77 for ; Tue, 1 Oct 2019 20:29:12 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 195FD15A1; Tue, 1 Oct 2019 11:29:12 -0700 (PDT) Received: from qc2400f-1.austin.arm.com (qc2400f-1.austin.arm.com [10.118.12.34]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 0DDF03F706; Tue, 1 Oct 2019 11:29:12 -0700 (PDT) From: Honnappa Nagarahalli To: bruce.richardson@intel.com, vladimir.medvedkin@intel.com, olivier.matz@6wind.com Cc: dev@dpdk.org, konstantin.ananyev@intel.com, stephen@networkplumber.org, paulmck@linux.ibm.com, Gavin.Hu@arm.com, Honnappa.Nagarahalli@arm.com, Dharmik.Thakkar@arm.com, Ruifeng.Wang@arm.com, nd@arm.com, Ruifeng Wang Date: Tue, 1 Oct 2019 13:28:56 -0500 Message-Id: <20191001182857.43867-3-honnappa.nagarahalli@arm.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20191001182857.43867-1-honnappa.nagarahalli@arm.com> References: <20190906094534.36060-1-ruifeng.wang@arm.com> <20191001182857.43867-1-honnappa.nagarahalli@arm.com> Subject: [dpdk-dev] [PATCH v3 2/3] app/test: add test case for LPM RCU integration X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Ruifeng Wang Add positive and negative tests for API rte_lpm_rcu_qsbr_add. Also test LPM library behavior when RCU QSBR is enabled. Signed-off-by: Ruifeng Wang Reviewed-by: Gavin Hu Reviewed-by: Honnappa Nagarahalli --- app/test/test_lpm.c | 152 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 1 deletion(-) diff --git a/app/test/test_lpm.c b/app/test/test_lpm.c index e969fe051..6882cae6a 100644 --- a/app/test/test_lpm.c +++ b/app/test/test_lpm.c @@ -8,6 +8,7 @@ #include #include +#include #include "test.h" #include "test_xmmt_ops.h" @@ -40,6 +41,8 @@ static int32_t test15(void); static int32_t test16(void); static int32_t test17(void); static int32_t test18(void); +static int32_t test19(void); +static int32_t test20(void); rte_lpm_test tests[] = { /* Test Cases */ @@ -61,7 +64,9 @@ rte_lpm_test tests[] = { test15, test16, test17, - test18 + test18, + test19, + test20 }; #define NUM_LPM_TESTS (sizeof(tests)/sizeof(tests[0])) @@ -1266,6 +1271,151 @@ test18(void) return PASS; } +/* + * rte_lpm_rcu_qsbr_add positive and negative tests. + * - Add RCU QSBR variable to LPM + * - Add another RCU QSBR variable to LPM + * - Check LPM attached RCU QSBR variable and FIFO queue + */ +int32_t +test19(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + size_t sz; + struct rte_rcu_qsbr *qsv; + struct rte_rcu_qsbr *qsv2; + int32_t status; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Create RCU QSBR variable */ + sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE); + qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket(NULL, sz, + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); + TEST_LPM_ASSERT(qsv != NULL); + + status = rte_rcu_qsbr_init(qsv, RTE_MAX_LCORE); + TEST_LPM_ASSERT(status == 0); + + /* Attach RCU QSBR to LPM table */ + status = rte_lpm_rcu_qsbr_add(lpm, qsv); + TEST_LPM_ASSERT(status == 0); + + /* Create and attach another RCU QSBR to LPM table */ + qsv2 = (struct rte_rcu_qsbr *)rte_zmalloc_socket(NULL, sz, + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); + TEST_LPM_ASSERT(qsv2 != NULL); + + status = rte_lpm_rcu_qsbr_add(lpm, qsv2); + TEST_LPM_ASSERT(status != 0); + + TEST_LPM_ASSERT(lpm->dq != NULL); + + rte_lpm_free(lpm); + rte_free(qsv); + rte_free(qsv2); + + return PASS; +} + +/* + * rte_lpm_rcu_qsbr_add functional test. + * - Create LPM which supports 1 tbl8 group at max + * - Add RCU QSBR variable to LPM + * - Add a rule with depth=28 (> 24) + * - Register a reader thread (not a real thread) + * - Reader lookup existing rule + * - Writer delete the rule + * - Reader lookup the rule + * - Writer re-add the rule (no available tbl8 group) + * - Reader report quiescent state and unregister + * - Writer re-add the rule + * - Reader lookup the rule + */ +int32_t +test20(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + size_t sz; + struct rte_rcu_qsbr *qsv; + int32_t status; + uint32_t ip, next_hop, next_hop_return; + uint8_t depth; + + config.max_rules = MAX_RULES; + config.number_tbl8s = 1; + config.flags = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Create RCU QSBR variable */ + sz = rte_rcu_qsbr_get_memsize(1); + qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket(NULL, sz, + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); + TEST_LPM_ASSERT(qsv != NULL); + + status = rte_rcu_qsbr_init(qsv, 1); + TEST_LPM_ASSERT(status == 0); + + /* Attach RCU QSBR to LPM table */ + status = rte_lpm_rcu_qsbr_add(lpm, qsv); + TEST_LPM_ASSERT(status == 0); + + ip = RTE_IPV4(192, 18, 100, 100); + depth = 28; + next_hop = 1; + status = rte_lpm_add(lpm, ip, depth, next_hop); + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(lpm->tbl24[ip>>8].valid_group); + + /* Register pseudo reader */ + status = rte_rcu_qsbr_thread_register(qsv, 0); + TEST_LPM_ASSERT(status == 0); + rte_rcu_qsbr_thread_online(qsv, 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop); + + /* Writer update */ + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(!lpm->tbl24[ip>>8].valid); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status != 0); + + status = rte_lpm_add(lpm, ip, depth, next_hop); + TEST_LPM_ASSERT(status != 0); + + /* Reader quiescent */ + rte_rcu_qsbr_quiescent(qsv, 0); + + status = rte_lpm_add(lpm, ip, depth, next_hop); + TEST_LPM_ASSERT(status == 0); + + rte_rcu_qsbr_thread_offline(qsv, 0); + status = rte_rcu_qsbr_thread_unregister(qsv, 0); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop); + + rte_lpm_free(lpm); + rte_free(qsv); + + return PASS; +} + /* * Do all unit tests. */ From patchwork Tue Oct 1 18:28:57 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Honnappa Nagarahalli X-Patchwork-Id: 60364 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 065151BEAD; Tue, 1 Oct 2019 20:29:20 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by dpdk.org (Postfix) with ESMTP id 0D9DA5B3A for ; Tue, 1 Oct 2019 20:29:13 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 327E215A2; Tue, 1 Oct 2019 11:29:12 -0700 (PDT) Received: from qc2400f-1.austin.arm.com (qc2400f-1.austin.arm.com [10.118.12.34]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 191FE3F918; Tue, 1 Oct 2019 11:29:12 -0700 (PDT) From: Honnappa Nagarahalli To: bruce.richardson@intel.com, vladimir.medvedkin@intel.com, olivier.matz@6wind.com Cc: dev@dpdk.org, konstantin.ananyev@intel.com, stephen@networkplumber.org, paulmck@linux.ibm.com, Gavin.Hu@arm.com, Honnappa.Nagarahalli@arm.com, Dharmik.Thakkar@arm.com, Ruifeng.Wang@arm.com, nd@arm.com, Honnappa Nagarahalli Date: Tue, 1 Oct 2019 13:28:57 -0500 Message-Id: <20191001182857.43867-4-honnappa.nagarahalli@arm.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20191001182857.43867-1-honnappa.nagarahalli@arm.com> References: <20190906094534.36060-1-ruifeng.wang@arm.com> <20191001182857.43867-1-honnappa.nagarahalli@arm.com> Subject: [dpdk-dev] [PATCH v3 3/3] test/lpm: add RCU integration performance tests X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add performance tests for RCU integration. The performance difference with and without RCU integration is very small (~1% to ~2%) on both Arm and x86 platforms. Signed-off-by: Honnappa Nagarahalli Reviewed-by: Gavin Hu Reviewed-by: Ruifeng Wang --- app/test/test_lpm_perf.c | 487 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 484 insertions(+), 3 deletions(-) diff --git a/app/test/test_lpm_perf.c b/app/test/test_lpm_perf.c index 77eea66ad..a9f02d983 100644 --- a/app/test/test_lpm_perf.c +++ b/app/test/test_lpm_perf.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2019 Arm Limited */ #include @@ -10,12 +11,28 @@ #include #include #include +#include #include #include +#include #include "test.h" #include "test_xmmt_ops.h" +struct rte_lpm *lpm; +static struct rte_rcu_qsbr *rv; +static volatile uint8_t writer_done; +static volatile uint32_t thr_id; +static rte_atomic64_t gwrite_cycles; +static rte_atomic64_t gwrites; +/* LPM APIs are not thread safe, use mutex to provide thread safety */ +static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* Report quiescent state interval every 8192 lookups. Larger critical + * sections in reader will result in writer polling multiple times. + */ +#define QSBR_REPORTING_INTERVAL 1024 + #define TEST_LPM_ASSERT(cond) do { \ if (!(cond)) { \ printf("Error at line %d: \n", __LINE__); \ @@ -24,6 +41,7 @@ } while(0) #define ITERATIONS (1 << 10) +#define RCU_ITERATIONS 10 #define BATCH_SIZE (1 << 12) #define BULK_SIZE 32 @@ -35,9 +53,13 @@ struct route_rule { }; struct route_rule large_route_table[MAX_RULE_NUM]; +/* Route table for routes with depth > 24 */ +struct route_rule large_ldepth_route_table[MAX_RULE_NUM]; static uint32_t num_route_entries; +static uint32_t num_ldepth_route_entries; #define NUM_ROUTE_ENTRIES num_route_entries +#define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entries enum { IP_CLASS_A, @@ -191,7 +213,7 @@ static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth) uint32_t ip_head_mask; uint32_t rule_num; uint32_t k; - struct route_rule *ptr_rule; + struct route_rule *ptr_rule, *ptr_ldepth_rule; if (ip_class == IP_CLASS_A) { /* IP Address class A */ fixed_bit_num = IP_HEAD_BIT_NUM_A; @@ -236,10 +258,20 @@ static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth) */ start = lrand48() & mask; ptr_rule = &large_route_table[num_route_entries]; + ptr_ldepth_rule = &large_ldepth_route_table[num_ldepth_route_entries]; for (k = 0; k < rule_num; k++) { ptr_rule->ip = (start << (RTE_LPM_MAX_DEPTH - depth)) | ip_head_mask; ptr_rule->depth = depth; + /* If the depth of the route is more than 24, store it + * in another table as well. + */ + if (depth > 24) { + ptr_ldepth_rule->ip = ptr_rule->ip; + ptr_ldepth_rule->depth = ptr_rule->depth; + ptr_ldepth_rule++; + num_ldepth_route_entries++; + } ptr_rule++; start = (start + step) & mask; } @@ -273,6 +305,7 @@ static void generate_large_route_rule_table(void) uint8_t depth; num_route_entries = 0; + num_ldepth_route_entries = 0; memset(large_route_table, 0, sizeof(large_route_table)); for (ip_class = IP_CLASS_A; ip_class <= IP_CLASS_C; ip_class++) { @@ -316,10 +349,454 @@ print_route_distribution(const struct route_rule *table, uint32_t n) printf("\n"); } +/* Check condition and return an error if true. */ +static uint16_t enabled_core_ids[RTE_MAX_LCORE]; +static unsigned int num_cores; + +/* Simple way to allocate thread ids in 0 to RTE_MAX_LCORE space */ +static inline uint32_t +alloc_thread_id(void) +{ + uint32_t tmp_thr_id; + + tmp_thr_id = __atomic_fetch_add(&thr_id, 1, __ATOMIC_RELAXED); + if (tmp_thr_id >= RTE_MAX_LCORE) + printf("Invalid thread id %u\n", tmp_thr_id); + + return tmp_thr_id; +} + +/* + * Reader thread using rte_lpm data structure without RCU. + */ +static int +test_lpm_reader(__attribute__((unused)) void *arg) +{ + int i; + uint32_t ip_batch[QSBR_REPORTING_INTERVAL]; + uint32_t next_hop_return = 0; + + do { + for (i = 0; i < QSBR_REPORTING_INTERVAL; i++) + ip_batch[i] = rte_rand(); + + for (i = 0; i < QSBR_REPORTING_INTERVAL; i++) + rte_lpm_lookup(lpm, ip_batch[i], &next_hop_return); + + } while (!writer_done); + + return 0; +} + +/* + * Reader thread using rte_lpm data structure with RCU. + */ +static int +test_lpm_rcu_qsbr_reader(__attribute__((unused)) void *arg) +{ + int i; + uint32_t thread_id = alloc_thread_id(); + uint32_t ip_batch[QSBR_REPORTING_INTERVAL]; + uint32_t next_hop_return = 0; + + /* Register this thread to report quiescent state */ + rte_rcu_qsbr_thread_register(rv, thread_id); + rte_rcu_qsbr_thread_online(rv, thread_id); + + do { + for (i = 0; i < QSBR_REPORTING_INTERVAL; i++) + ip_batch[i] = rte_rand(); + + for (i = 0; i < QSBR_REPORTING_INTERVAL; i++) + rte_lpm_lookup(lpm, ip_batch[i], &next_hop_return); + + /* Update quiescent state */ + rte_rcu_qsbr_quiescent(rv, thread_id); + } while (!writer_done); + + rte_rcu_qsbr_thread_offline(rv, thread_id); + rte_rcu_qsbr_thread_unregister(rv, thread_id); + + return 0; +} + +/* + * Writer thread using rte_lpm data structure with RCU. + */ +static int +test_lpm_rcu_qsbr_writer(__attribute__((unused)) void *arg) +{ + unsigned int i, j, si, ei; + uint64_t begin, total_cycles; + uint8_t core_id = (uint8_t)((uintptr_t)arg); + uint32_t next_hop_add = 0xAA; + + /* 2 writer threads are used */ + if (core_id % 2 == 0) { + si = 0; + ei = NUM_LDEPTH_ROUTE_ENTRIES / 2; + } else { + si = NUM_LDEPTH_ROUTE_ENTRIES / 2; + ei = NUM_LDEPTH_ROUTE_ENTRIES; + } + + /* Measure add/delete. */ + begin = rte_rdtsc_precise(); + for (i = 0; i < RCU_ITERATIONS; i++) { + /* Add all the entries */ + for (j = si; j < ei; j++) { + pthread_mutex_lock(&lpm_mutex); + if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) != 0) { + printf("Failed to add iteration %d, route# %d\n", + i, j); + } + pthread_mutex_unlock(&lpm_mutex); + } + + /* Delete all the entries */ + for (j = si; j < ei; j++) { + pthread_mutex_lock(&lpm_mutex); + if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) != 0) { + printf("Failed to delete iteration %d, route# %d\n", + i, j); + } + pthread_mutex_unlock(&lpm_mutex); + } + } + + total_cycles = rte_rdtsc_precise() - begin; + + rte_atomic64_add(&gwrite_cycles, total_cycles); + rte_atomic64_add(&gwrites, + 2 * NUM_LDEPTH_ROUTE_ENTRIES * RCU_ITERATIONS); + + return 0; +} + +/* + * Functional test: + * 2 writers, rest are readers + */ +static int +test_lpm_rcu_perf_multi_writer(void) +{ + struct rte_lpm_config config; + size_t sz; + unsigned int i; + uint16_t core_id; + + if (rte_lcore_count() < 3) { + printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n"); + return TEST_SKIPPED; + } + + num_cores = 0; + RTE_LCORE_FOREACH_SLAVE(core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + + printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n", + num_cores - 2); + + /* Create LPM table */ + config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; + config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES; + config.flags = 0; + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Init RCU variable */ + sz = rte_rcu_qsbr_get_memsize(num_cores); + rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz, + RTE_CACHE_LINE_SIZE); + rte_rcu_qsbr_init(rv, num_cores); + + /* Assign the RCU variable to LPM */ + if (rte_lpm_rcu_qsbr_add(lpm, rv) != 0) { + printf("RCU variable assignment failed\n"); + goto error; + } + + writer_done = 0; + rte_atomic64_init(&gwrite_cycles); + rte_atomic64_init(&gwrites); + rte_atomic64_clear(&gwrite_cycles); + rte_atomic64_clear(&gwrites); + + __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST); + + /* Launch reader threads */ + for (i = 2; i < num_cores; i++) + rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL, + enabled_core_ids[i]); + + /* Launch writer threads */ + for (i = 0; i < 2; i++) + rte_eal_remote_launch(test_lpm_rcu_qsbr_writer, + (void *)(uintptr_t)i, + enabled_core_ids[i]); + + /* Wait for writer threads */ + for (i = 0; i < 2; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto error; + + printf("Total LPM Adds: %d\n", + 2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES); + printf("Total LPM Deletes: %d\n", + 2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES); + printf("Average LPM Add/Del: %lu cycles\n", + rte_atomic64_read(&gwrite_cycles) / rte_atomic64_read(&gwrites) + ); + + /* Wait and check return value from reader threads */ + writer_done = 1; + for (i = 2; i < num_cores; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto error; + + rte_lpm_free(lpm); + rte_free(rv); + lpm = NULL; + rv = NULL; + + /* Test without RCU integration */ + printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n", + num_cores - 2); + + /* Create LPM table */ + config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; + config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES; + config.flags = 0; + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + writer_done = 0; + rte_atomic64_init(&gwrite_cycles); + rte_atomic64_init(&gwrites); + rte_atomic64_clear(&gwrite_cycles); + rte_atomic64_clear(&gwrites); + __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST); + + /* Launch reader threads */ + for (i = 2; i < num_cores; i++) + rte_eal_remote_launch(test_lpm_reader, NULL, + enabled_core_ids[i]); + + /* Launch writer threads */ + for (i = 0; i < 2; i++) + rte_eal_remote_launch(test_lpm_rcu_qsbr_writer, + (void *)(uintptr_t)i, + enabled_core_ids[i]); + + /* Wait for writer threads */ + for (i = 0; i < 2; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto error; + + printf("Total LPM Adds: %d\n", + 2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES); + printf("Total LPM Deletes: %d\n", + 2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES); + printf("Average LPM Add/Del: %lu cycles\n", + rte_atomic64_read(&gwrite_cycles) / rte_atomic64_read(&gwrites) + ); + + writer_done = 1; + /* Wait and check return value from reader threads */ + for (i = 2; i < num_cores; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto error; + + rte_lpm_free(lpm); + + return 0; + +error: + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + + rte_lpm_free(lpm); + rte_free(rv); + + return -1; +} + +/* + * Functional test: + * Single writer, rest are readers + */ +static int +test_lpm_rcu_perf(void) +{ + struct rte_lpm_config config; + uint64_t begin, total_cycles; + size_t sz; + unsigned int i, j; + uint16_t core_id; + uint32_t next_hop_add = 0xAA; + + if (rte_lcore_count() < 2) { + printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n"); + return TEST_SKIPPED; + } + + num_cores = 0; + RTE_LCORE_FOREACH_SLAVE(core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + + printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n", + num_cores); + + /* Create LPM table */ + config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; + config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES; + config.flags = 0; + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Init RCU variable */ + sz = rte_rcu_qsbr_get_memsize(num_cores); + rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz, + RTE_CACHE_LINE_SIZE); + rte_rcu_qsbr_init(rv, num_cores); + + /* Assign the RCU variable to LPM */ + if (rte_lpm_rcu_qsbr_add(lpm, rv) != 0) { + printf("RCU variable assignment failed\n"); + goto error; + } + + writer_done = 0; + __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST); + + /* Launch reader threads */ + for (i = 0; i < num_cores; i++) + rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL, + enabled_core_ids[i]); + + /* Measure add/delete. */ + begin = rte_rdtsc_precise(); + for (i = 0; i < RCU_ITERATIONS; i++) { + /* Add all the entries */ + for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) + if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) != 0) { + printf("Failed to add iteration %d, route# %d\n", + i, j); + goto error; + } + + /* Delete all the entries */ + for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) + if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) != 0) { + printf("Failed to delete iteration %d, route# %d\n", + i, j); + goto error; + } + } + total_cycles = rte_rdtsc_precise() - begin; + + printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES); + printf("Total LPM Deletes: %d\n", + ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES); + printf("Average LPM Add/Del: %g cycles\n", + (double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS)); + + writer_done = 1; + /* Wait and check return value from reader threads */ + for (i = 0; i < num_cores; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + goto error; + + rte_lpm_free(lpm); + rte_free(rv); + lpm = NULL; + rv = NULL; + + /* Test without RCU integration */ + printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n", + num_cores); + + /* Create LPM table */ + config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; + config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES; + config.flags = 0; + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + writer_done = 0; + __atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST); + + /* Launch reader threads */ + for (i = 0; i < num_cores; i++) + rte_eal_remote_launch(test_lpm_reader, NULL, + enabled_core_ids[i]); + + /* Measure add/delete. */ + begin = rte_rdtsc_precise(); + for (i = 0; i < RCU_ITERATIONS; i++) { + /* Add all the entries */ + for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) + if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) != 0) { + printf("Failed to add iteration %d, route# %d\n", + i, j); + goto error; + } + + /* Delete all the entries */ + for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) + if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) != 0) { + printf("Failed to delete iteration %d, route# %d\n", + i, j); + goto error; + } + } + total_cycles = rte_rdtsc_precise() - begin; + + printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES); + printf("Total LPM Deletes: %d\n", + ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES); + printf("Average LPM Add/Del: %g cycles\n", + (double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS)); + + writer_done = 1; + /* Wait and check return value from reader threads */ + for (i = 0; i < num_cores; i++) + if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) + printf("Warning: lcore %u not finished.\n", + enabled_core_ids[i]); + + rte_lpm_free(lpm); + + return 0; + +error: + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + + rte_lpm_free(lpm); + rte_free(rv); + + return -1; +} + static int test_lpm_perf(void) { - struct rte_lpm *lpm = NULL; struct rte_lpm_config config; config.max_rules = 2000000; @@ -343,7 +820,7 @@ test_lpm_perf(void) lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); TEST_LPM_ASSERT(lpm != NULL); - /* Measue add. */ + /* Measure add. */ begin = rte_rdtsc(); for (i = 0; i < NUM_ROUTE_ENTRIES; i++) { @@ -478,6 +955,10 @@ test_lpm_perf(void) rte_lpm_delete_all(lpm); rte_lpm_free(lpm); + test_lpm_rcu_perf(); + + test_lpm_rcu_perf_multi_writer(); + return 0; }