From patchwork Sat Dec 22 02:14:19 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Honnappa Nagarahalli X-Patchwork-Id: 49250 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 63D361BE66; Sat, 22 Dec 2018 03:14:57 +0100 (CET) Received: from foss.arm.com (foss.arm.com [217.140.101.70]) by dpdk.org (Postfix) with ESMTP id 5E5761BE66 for ; Sat, 22 Dec 2018 03:14:55 +0100 (CET) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 9B278EBD; Fri, 21 Dec 2018 18:14:54 -0800 (PST) Received: from qc2400f-1.austin.arm.com (qc2400f-1.austin.arm.com [10.118.12.24]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 131A23F575; Fri, 21 Dec 2018 18:14:54 -0800 (PST) From: Honnappa Nagarahalli To: dev@dpdk.org, konstantin.ananyev@intel.com, stephen@networkplumber.org, paulmck@linux.ibm.com, honnappa.nagarahalli@arm.com Cc: gavin.hu@arm.com, dharmik.thakkar@arm.com, nd@arm.com Date: Fri, 21 Dec 2018 20:14:19 -0600 Message-Id: <20181222021420.5114-2-honnappa.nagarahalli@arm.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20181222021420.5114-1-honnappa.nagarahalli@arm.com> References: <20181122033055.3431-1-honnappa.nagarahalli@arm.com> <20181222021420.5114-1-honnappa.nagarahalli@arm.com> Subject: [dpdk-dev] [RFC v2 1/2] rcu: add RCU library supporting QSBR mechanism X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add RCU library supporting quiescent state based memory reclamation method. This library helps identify the quiescent state of the reader threads so that the writers can free the memory associated with the lock less data structures. Signed-off-by: Honnappa Nagarahalli Reviewed-by: Steve Capper Reviewed-by: Gavin Hu --- config/common_base | 6 + lib/Makefile | 2 + lib/librte_rcu/Makefile | 23 +++ lib/librte_rcu/meson.build | 5 + lib/librte_rcu/rte_rcu_qsbr.c | 63 ++++++ lib/librte_rcu/rte_rcu_qsbr.h | 321 +++++++++++++++++++++++++++++ lib/librte_rcu/rte_rcu_version.map | 8 + lib/meson.build | 2 +- mk/rte.app.mk | 1 + 9 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 lib/librte_rcu/Makefile create mode 100644 lib/librte_rcu/meson.build create mode 100644 lib/librte_rcu/rte_rcu_qsbr.c create mode 100644 lib/librte_rcu/rte_rcu_qsbr.h create mode 100644 lib/librte_rcu/rte_rcu_version.map diff --git a/config/common_base b/config/common_base index d12ae98bc..e148549d8 100644 --- a/config/common_base +++ b/config/common_base @@ -792,6 +792,12 @@ CONFIG_RTE_LIBRTE_LATENCY_STATS=y # CONFIG_RTE_LIBRTE_TELEMETRY=n +# +# Compile librte_rcu +# +CONFIG_RTE_LIBRTE_RCU=y +CONFIG_RTE_LIBRTE_RCU_DEBUG=n + # # Compile librte_lpm # diff --git a/lib/Makefile b/lib/Makefile index b7370ef97..b674662c8 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -108,6 +108,8 @@ DIRS-$(CONFIG_RTE_LIBRTE_BPF) += librte_bpf DEPDIRS-librte_bpf := librte_eal librte_mempool librte_mbuf librte_ethdev DIRS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += librte_telemetry DEPDIRS-librte_telemetry := librte_eal librte_metrics librte_ethdev +DIRS-$(CONFIG_RTE_LIBRTE_RCU) += librte_rcu +DEPDIRS-librte_rcu := librte_eal ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y) DIRS-$(CONFIG_RTE_LIBRTE_KNI) += librte_kni diff --git a/lib/librte_rcu/Makefile b/lib/librte_rcu/Makefile new file mode 100644 index 000000000..6aa677bd1 --- /dev/null +++ b/lib/librte_rcu/Makefile @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Arm Limited + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_rcu.a + +CFLAGS += -DALLOW_EXPERIMENTAL_API +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal + +EXPORT_MAP := rte_rcu_version.map + +LIBABIVER := 1 + +# all source are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_RCU) := rte_rcu_qsbr.c + +# install includes +SYMLINK-$(CONFIG_RTE_LIBRTE_RCU)-include := rte_rcu_qsbr.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_rcu/meson.build b/lib/librte_rcu/meson.build new file mode 100644 index 000000000..c009ae4b7 --- /dev/null +++ b/lib/librte_rcu/meson.build @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Arm Limited + +sources = files('rte_rcu_qsbr.c') +headers = files('rte_rcu_qsbr.h') diff --git a/lib/librte_rcu/rte_rcu_qsbr.c b/lib/librte_rcu/rte_rcu_qsbr.c new file mode 100644 index 000000000..3c2577ee2 --- /dev/null +++ b/lib/librte_rcu/rte_rcu_qsbr.c @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 2018 Arm Limited + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_rcu_qsbr.h" + +/* Initialize a quiescent state variable */ +void __rte_experimental +rte_rcu_qsbr_init(struct rte_rcu_qsbr *v) +{ + memset(v, 0, sizeof(struct rte_rcu_qsbr)); +} + +/* Dump the details of a single quiescent state variable to a file. */ +void __rte_experimental +rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) +{ + uint64_t bmap; + uint32_t i, t; + + RTE_ASSERT(v == NULL || f == NULL); + + fprintf(f, "\nQuiescent State Variable @%p\n", v); + + fprintf(f, " Registered thread ID mask = 0x"); + for (i = 0; i < RTE_QSBR_BIT_MAP_ELEMS; i++) + fprintf(f, "%lx", __atomic_load_n(&v->reg_thread_id[i], + __ATOMIC_ACQUIRE)); + fprintf(f, "\n"); + + fprintf(f, " Token = %lu\n", + __atomic_load_n(&v->token, __ATOMIC_ACQUIRE)); + + fprintf(f, "Quiescent State Counts for readers:\n"); + for (i = 0; i < RTE_QSBR_BIT_MAP_ELEMS; i++) { + bmap = __atomic_load_n(&v->reg_thread_id[i], + __ATOMIC_ACQUIRE); + while (bmap) { + t = __builtin_ctzl(bmap); + fprintf(f, "thread ID = %d, count = %lu\n", t, + __atomic_load_n(&v->w[i].cnt, + __ATOMIC_RELAXED)); + bmap &= ~(1UL << t); + } + } +} diff --git a/lib/librte_rcu/rte_rcu_qsbr.h b/lib/librte_rcu/rte_rcu_qsbr.h new file mode 100644 index 000000000..c818e77fd --- /dev/null +++ b/lib/librte_rcu/rte_rcu_qsbr.h @@ -0,0 +1,321 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018 Arm Limited + */ + +#ifndef _RTE_RCU_QSBR_H_ +#define _RTE_RCU_QSBR_H_ + +/** + * @file + * RTE Quiescent State Based Reclamation (QSBR) + * + * Quiescent State (QS) is any point in the thread execution + * where the thread does not hold a reference to shared memory. + * A critical section for a data structure can be a quiescent state for + * another data structure. + * + * This library provides the ability to identify quiescent state. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include + +/**< Maximum number of reader threads supported. */ +#define RTE_RCU_MAX_THREADS 128 + +#if !RTE_IS_POWER_OF_2(RTE_RCU_MAX_THREADS) +#error RTE_RCU_MAX_THREADS must be a power of 2 +#endif + +/**< Number of array elements required for the bit-map */ +#define RTE_QSBR_BIT_MAP_ELEMS (RTE_RCU_MAX_THREADS/(sizeof(uint64_t) * 8)) + +/* Thread IDs are stored as a bitmap of 64b element array. Given thread id + * needs to be converted to index into the array and the id within + * the array element. + */ +#define RTE_QSBR_THR_INDEX_SHIFT 6 +#define RTE_QSBR_THR_ID_MASK 0x3f + +/* Worker thread counter */ +struct rte_rcu_qsbr_cnt { + uint64_t cnt; /**< Quiescent state counter. */ +} __rte_cache_aligned; + +/** + * RTE thread Quiescent State structure. + */ +struct rte_rcu_qsbr { + uint64_t reg_thread_id[RTE_QSBR_BIT_MAP_ELEMS] __rte_cache_aligned; + /**< Registered reader thread IDs - reader threads reporting + * on this QS variable represented in a bit map. + */ + + uint64_t token __rte_cache_aligned; + /**< Counter to allow for multiple simultaneous QS queries */ + + struct rte_rcu_qsbr_cnt w[RTE_RCU_MAX_THREADS] __rte_cache_aligned; + /**< QS counter for each reader thread, counts upto + * current value of token. + */ +} __rte_cache_aligned; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Initialize a Quiescent State (QS) variable. + * + * @param v + * QS variable + * + */ +void __rte_experimental +rte_rcu_qsbr_init(struct rte_rcu_qsbr *v); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Add a reader thread, to the list of threads reporting their quiescent + * state on a QS variable. + * + * This is implemented as a lock-free function. It is multi-thread + * safe. + * Any reader thread that wants to report its quiescent state must + * call this API before calling rte_rcu_qsbr_update. This can be called + * during initialization or as part of the packet processing loop. + * Any ongoing QS queries may wait for the status from this registered + * thread. + * + * @param v + * QS variable + * @param thread_id + * Reader thread with this thread ID will report its quiescent state on + * the QS variable. + */ +static __rte_always_inline void __rte_experimental +rte_rcu_qsbr_register_thread(struct rte_rcu_qsbr *v, unsigned int thread_id) +{ + unsigned int i, id; + + RTE_ASSERT(v == NULL || thread_id >= RTE_RCU_MAX_THREADS); + + id = thread_id & RTE_QSBR_THR_ID_MASK; + i = thread_id >> RTE_QSBR_THR_INDEX_SHIFT; + + /* Worker thread has to count the quiescent states + * only from the current value of token. + * __atomic_store_n(cnt, __ATOMIC_RELAXED) is used to ensure + * 'cnt' (64b) is accessed atomically. + */ + __atomic_store_n(&v->w[thread_id].cnt, + __atomic_load_n(&v->token, __ATOMIC_ACQUIRE), + __ATOMIC_RELAXED); + + /* Release the store to initial TQS count so that readers + * can use it immediately after this function returns. + */ + __atomic_fetch_or(&v->reg_thread_id[i], 1UL << id, __ATOMIC_RELEASE); +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Remove a reader thread, from the list of threads reporting their + * quiescent state on a QS variable. + * + * This is implemented as a lock-free function. It is multi-thread safe. + * This API can be called from the reader threads during shutdown. + * Ongoing QS queries will stop waiting for the status from this + * unregistered reader thread. + * + * @param v + * QS variable + * @param thread_id + * Reader thread with this thread ID will stop reporting its quiescent + * state on the QS variable. + */ +static __rte_always_inline void __rte_experimental +rte_rcu_qsbr_unregister_thread(struct rte_rcu_qsbr *v, unsigned int thread_id) +{ + unsigned int i, id; + + RTE_ASSERT(v == NULL || thread_id >= RTE_RCU_MAX_THREADS); + + id = thread_id & RTE_QSBR_THR_ID_MASK; + i = thread_id >> RTE_QSBR_THR_INDEX_SHIFT; + + /* Make sure the removal of the thread from the list of + * reporting threads is visible before the thread + * does anything else. + */ + __atomic_fetch_and(&v->reg_thread_id[i], + ~(1UL << id), __ATOMIC_RELEASE); +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Trigger the reader threads to report the quiescent state + * status. + * + * This is implemented as a lock-free function. It is multi-thread + * safe and can be called from worker threads. + * + * @param v + * TQS variable + * @param n + * Expected number of times the quiescent state is entered + * @param t + * - If successful, this is the token for this call of the API. + * This should be passed to rte_rcu_qsbr_check API. + */ +static __rte_always_inline void __rte_experimental +rte_rcu_qsbr_start(struct rte_rcu_qsbr *v, unsigned int n, uint64_t *t) +{ + RTE_ASSERT(v == NULL || t == NULL); + + /* This store release will ensure that changes to any data + * structure are visible to the workers before the token + * update is visible. + */ + *t = __atomic_add_fetch(&v->token, n, __ATOMIC_RELEASE); +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Update quiescent state for a reader thread. + * + * This is implemented as a lock-free function. It is multi-thread safe. + * All the reader threads registered to report their quiescent state + * on the QS variable must call this API. + * + * @param v + * QS variable + */ +static __rte_always_inline void __rte_experimental +rte_rcu_qsbr_update(struct rte_rcu_qsbr *v, unsigned int thread_id) +{ + uint64_t t; + + RTE_ASSERT(v == NULL || thread_id >= RTE_RCU_MAX_THREADS); + + /* Load the token before the reader thread loads any other + * (lock-free) data structure. This ensures that updates + * to the data structures are visible if the update + * to token is visible. + */ + t = __atomic_load_n(&v->token, __ATOMIC_ACQUIRE); + + /* Relaxed load/store on the counter is enough as we are + * reporting an already completed quiescent state. + * __atomic_load_n(cnt, __ATOMIC_RELAXED) is used as 'cnt' (64b) + * is accessed atomically. + * Copy the current token value. This will end grace period + * of multiple concurrent writers. + */ + if (__atomic_load_n(&v->w[thread_id].cnt, __ATOMIC_RELAXED) != t) + __atomic_store_n(&v->w[thread_id].cnt, t, __ATOMIC_RELAXED); +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Checks if all the reader threads have entered the quiescent state + * 'n' number of times. 'n' is provided in rte_rcu_qsbr_start API. + * + * This is implemented as a lock-free function. It is multi-thread + * safe and can be called from the worker threads as well. + * + * @param v + * QS variable + * @param t + * Token returned by rte_rcu_qsbr_start API + * @param wait + * If true, block till all the reader threads have completed entering + * the quiescent state 'n' number of times + * @return + * - 0 if all reader threads have NOT passed through specified number + * of quiescent states. + * - 1 if all reader threads have passed through specified number + * of quiescent states. + */ +static __rte_always_inline int __rte_experimental +rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait) +{ + uint32_t i, j, id; + uint64_t bmap; + + RTE_ASSERT(v == NULL); + + i = 0; + do { + /* Load the current registered thread bit map before + * loading the reader thread quiescent state counters. + */ + bmap = __atomic_load_n(&v->reg_thread_id[i], __ATOMIC_ACQUIRE); + id = i << RTE_QSBR_THR_INDEX_SHIFT; + + while (bmap) { + j = __builtin_ctzl(bmap); + + /* __atomic_store_n(cnt, __ATOMIC_RELAXED) + * is used to ensure 'cnt' (64b) is accessed + * atomically. + */ + if (unlikely(__atomic_load_n(&v->w[id + j].cnt, + __ATOMIC_RELAXED) < t)) { + /* This thread is not in QS */ + if (!wait) + return 0; + + /* Loop till this thread enters QS */ + rte_pause(); + continue; + } + + bmap &= ~(1UL << j); + } + + i++; + } while (i < RTE_QSBR_BIT_MAP_ELEMS); + + return 1; +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Dump the details of a single QS variables to a file. + * + * It is NOT multi-thread safe. + * + * @param f + * A pointer to a file for output + * @param v + * QS variable + */ +void __rte_experimental +rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_RCU_QSBR_H_ */ diff --git a/lib/librte_rcu/rte_rcu_version.map b/lib/librte_rcu/rte_rcu_version.map new file mode 100644 index 000000000..0df2071be --- /dev/null +++ b/lib/librte_rcu/rte_rcu_version.map @@ -0,0 +1,8 @@ +EXPERIMENTAL { + global: + + rte_rcu_qsbr_init; + rte_rcu_qsbr_dump; + + local: *; +}; diff --git a/lib/meson.build b/lib/meson.build index bb7f443f9..d0e49d4e1 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -21,7 +21,7 @@ libraries = [ 'compat', # just a header, used for versioning 'gro', 'gso', 'ip_frag', 'jobstats', 'kni', 'latencystats', 'lpm', 'member', 'meter', 'power', 'pdump', 'rawdev', - 'reorder', 'sched', 'security', 'vhost', + 'reorder', 'sched', 'security', 'rcu', 'vhost', # add pkt framework libs which use other libs from above 'port', 'table', 'pipeline', # flow_classify lib depends on pkt framework table lib diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 3ebc4e64c..d4a1d436a 100644 --- a/mk/rte.app.mk +++ b/mk/rte.app.mk @@ -92,6 +92,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrte_eal _LDLIBS-$(CONFIG_RTE_LIBRTE_CMDLINE) += -lrte_cmdline _LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER) += -lrte_reorder _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrte_sched +_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcu ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y) _LDLIBS-$(CONFIG_RTE_LIBRTE_KNI) += -lrte_kni From patchwork Sat Dec 22 02:14:20 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Honnappa Nagarahalli X-Patchwork-Id: 49251 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id D82DE1BE60; Sat, 22 Dec 2018 03:15:09 +0100 (CET) Received: from foss.arm.com (usa-sjc-mx-foss1.foss.arm.com [217.140.101.70]) by dpdk.org (Postfix) with ESMTP id A86D91BE7A for ; Sat, 22 Dec 2018 03:15:07 +0100 (CET) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1FEF2EBD; Fri, 21 Dec 2018 18:15:07 -0800 (PST) Received: from qc2400f-1.austin.arm.com (qc2400f-1.austin.arm.com [10.118.12.24]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 77A1E3F575; Fri, 21 Dec 2018 18:15:06 -0800 (PST) From: Honnappa Nagarahalli To: dev@dpdk.org, konstantin.ananyev@intel.com, stephen@networkplumber.org, paulmck@linux.ibm.com, honnappa.nagarahalli@arm.com Cc: gavin.hu@arm.com, dharmik.thakkar@arm.com, nd@arm.com, Malvika Gupta Date: Fri, 21 Dec 2018 20:14:20 -0600 Message-Id: <20181222021420.5114-3-honnappa.nagarahalli@arm.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20181222021420.5114-1-honnappa.nagarahalli@arm.com> References: <20181122033055.3431-1-honnappa.nagarahalli@arm.com> <20181222021420.5114-1-honnappa.nagarahalli@arm.com> Subject: [dpdk-dev] [RFC v2 2/2] test/rcu_qsbr: add API and functional tests X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Dharmik Thakkar Add API positive/negative test cases and functional tests. Signed-off-by: Malvika Gupta Signed-off-by: Dharmik Thakkar Signed-off-by: Honnappa Nagarahalli Reviewed-by: Gavin Hu --- test/test/Makefile | 2 + test/test/autotest_data.py | 6 + test/test/meson.build | 5 +- test/test/test_rcu_qsbr.c | 801 +++++++++++++++++++++++++++++++++++++ 4 files changed, 813 insertions(+), 1 deletion(-) create mode 100644 test/test/test_rcu_qsbr.c diff --git a/test/test/Makefile b/test/test/Makefile index ab4fec34a..dfc0325e4 100644 --- a/test/test/Makefile +++ b/test/test/Makefile @@ -207,6 +207,8 @@ SRCS-$(CONFIG_RTE_LIBRTE_KVARGS) += test_kvargs.c SRCS-$(CONFIG_RTE_LIBRTE_BPF) += test_bpf.c +SRCS-$(CONFIG_RTE_LIBRTE_RCU) += test_rcu_qsbr.c + CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 diff --git a/test/test/autotest_data.py b/test/test/autotest_data.py index 0fb7866db..cbd1f94ad 100644 --- a/test/test/autotest_data.py +++ b/test/test/autotest_data.py @@ -676,6 +676,12 @@ "Func": default_autotest, "Report": None, }, + { + "Name": "RCU QSBR autotest", + "Command": "rcu_qsbr_autotest", + "Func": default_autotest, + "Report": None, + }, # # Please always make sure that ring_perf is the last test! # diff --git a/test/test/meson.build b/test/test/meson.build index 554e9945f..3be21be27 100644 --- a/test/test/meson.build +++ b/test/test/meson.build @@ -100,6 +100,7 @@ test_sources = files('commands.c', 'test_timer.c', 'test_timer_perf.c', 'test_timer_racecond.c', + 'test_rcu_qsbr.c', 'test_version.c', 'virtual_pmd.c' ) @@ -122,7 +123,8 @@ test_deps = ['acl', 'port', 'reorder', 'ring', - 'timer' + 'timer', + 'rcu' ] test_names = [ @@ -228,6 +230,7 @@ test_names = [ 'timer_autotest', 'timer_perf__autotest', 'timer_racecond_autotest', + 'rcu_qsbr_autotest', 'user_delay_us', 'version_autotest', ] diff --git a/test/test/test_rcu_qsbr.c b/test/test/test_rcu_qsbr.c new file mode 100644 index 000000000..c9efd3e21 --- /dev/null +++ b/test/test/test_rcu_qsbr.c @@ -0,0 +1,801 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018 Arm Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test.h" + +/* Check condition and return an error if true. */ +#define RCU_QSBR_RETURN_IF_ERROR(cond, str, ...) do { \ + if (cond) { \ + printf("ERROR file %s, line %d: " str "\n", __FILE__, \ + __LINE__, ##__VA_ARGS__); \ + return -1; \ + } \ +} while (0) + +#define RTE_RCU_MAX_LCORE 64 +uint16_t enabled_core_ids[RTE_RCU_MAX_LCORE]; +uint8_t num_cores; +uint16_t num_1qs = 1; /* Number of quiescent states = 1 */ +uint16_t num_2qs = 2; /* Number of quiescent states = 2 */ +uint16_t num_3qs = 3; /* Number of quiescent states = 3 */ + +static uint32_t *keys; +#define TOTAL_ENTRY (1024 * 8) +#define COUNTER_VALUE 4096 +uint32_t *hash_data[RTE_RCU_MAX_LCORE][TOTAL_ENTRY]; +uint8_t writer_done; + +struct rte_rcu_qsbr t[RTE_RCU_MAX_LCORE]; +struct rte_hash *h[RTE_RCU_MAX_LCORE]; +char hash_name[RTE_RCU_MAX_LCORE][8]; + +static inline int +get_enabled_cores_mask(void) +{ + uint16_t core_id; + uint32_t max_cores = rte_lcore_count(); + if (max_cores > RTE_RCU_MAX_LCORE) { + printf("Number of cores exceed %d\n", RTE_RCU_MAX_LCORE); + return -1; + } + + core_id = 0; + num_cores = 0; + RTE_LCORE_FOREACH_SLAVE(core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + + return 0; +} + +/* + * rte_rcu_qsbr_register_thread: Add a reader thread, to the list of threads + * reporting their quiescent state on a QS variable. + */ +static int +test_rcu_qsbr_register_thread(void) +{ + printf("\nTest rte_rcu_qsbr_register_thread()\n"); + + rte_rcu_qsbr_init(&t[0]); + + rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[0]); + return 0; +} + +/* + * rte_rcu_qsbr_unregister_thread: Remove a reader thread, from the list of + * threads reporting their quiescent state on a QS variable. + */ +static int +test_rcu_qsbr_unregister_thread(void) +{ + int i, j, ret; + uint64_t token; + uint8_t num_threads[3] = {1, RTE_RCU_MAX_THREADS, 1}; + + printf("\nTest rte_rcu_qsbr_unregister_thread()\n"); + + rte_rcu_qsbr_init(&t[0]); + + rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[0]); + + /* Find first disabled core */ + for (i = 0; i < RTE_RCU_MAX_LCORE; i++) { + if (enabled_core_ids[i] == 0) + break; + } + /* Test with disabled lcore */ + rte_rcu_qsbr_unregister_thread(&t[0], i); + + /* Test with enabled lcore */ + rte_rcu_qsbr_unregister_thread(&t[0], enabled_core_ids[0]); + + /* + * Test with different thread_ids: + * 1 - thread_id = 0 + * 2 - All possible thread_ids, from 0 to RTE_RCU_MAX_THREADS + * 3 - thread_id = RTE_RCU_MAX_THREADS - 1 + */ + for (j = 0; j < 3; j++) { + rte_rcu_qsbr_init(&t[0]); + + for (i = 0; i < num_threads[j]; i++) + rte_rcu_qsbr_register_thread(&t[0], + (j == 2) ? (RTE_RCU_MAX_THREADS - 1) : i); + + rte_rcu_qsbr_start(&t[0], 1, &token); + RCU_QSBR_RETURN_IF_ERROR((token != 1), "QSBR Start"); + /* Update quiescent state counter */ + for (i = 0; i < num_threads[j]; i++) { + /* Skip one update */ + if (i == 72) + continue; + rte_rcu_qsbr_update(&t[0], + (j == 2) ? (RTE_RCU_MAX_THREADS - 1) : i); + } + + if (j == 1) { + /* Validate the updates */ + ret = rte_rcu_qsbr_check(&t[0], token, false); + RCU_QSBR_RETURN_IF_ERROR((ret == 1), "Non-blocking QSBR check"); + /* Update the previously skipped thread */ + rte_rcu_qsbr_update(&t[0], 72); + } + + /* Validate the updates */ + ret = rte_rcu_qsbr_check(&t[0], token, false); + RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Non-blocking QSBR check"); + + for (i = 0; i < num_threads[j]; i++) + rte_rcu_qsbr_unregister_thread(&t[0], + (j == 2) ? (RTE_RCU_MAX_THREADS - 1) : i); + + /* Check with no thread registered */ + ret = rte_rcu_qsbr_check(&t[0], token, true); + RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Blocking QSBR check"); + } + return 0; +} + +/* + * rte_rcu_qsbr_start: Trigger the worker threads to report the quiescent state + * status. + */ +static int +test_rcu_qsbr_start(void) +{ + uint64_t token; + int i; + + printf("\nTest rte_rcu_qsbr_start()\n"); + + rte_rcu_qsbr_init(&t[0]); + + for (i = 0; i < 3; i++) + rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]); + + rte_rcu_qsbr_start(&t[0], 1, &token); + RCU_QSBR_RETURN_IF_ERROR((token != 1), "QSBR Start"); + return 0; +} + +/* + * rte_rcu_qsbr_check: Checks if all the worker threads have entered the queis- + * cent state 'n' number of times. 'n' is provided in rte_rcu_qsbr_start API. + */ +static int +test_rcu_qsbr_check(void) +{ + int i, ret; + uint64_t token; + + printf("\nTest rte_rcu_qsbr_check()\n"); + + rte_rcu_qsbr_init(&t[0]); + + rte_rcu_qsbr_start(&t[0], 1, &token); + RCU_QSBR_RETURN_IF_ERROR((token != 1), "QSBR Start"); + + ret = rte_rcu_qsbr_check(&t[0], 0, true); + RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Token = 0"); + + ret = rte_rcu_qsbr_check(&t[0], token, true); + RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Blocking QSBR check"); + + for (i = 0; i < 3; i++) + rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]); + + ret = rte_rcu_qsbr_check(&t[0], token, false); + RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Non-blocking QSBR check"); + + rte_rcu_qsbr_start(&t[0], 1, &token); + RCU_QSBR_RETURN_IF_ERROR((token != 2), "QSBR Start"); + + ret = rte_rcu_qsbr_check(&t[0], token, false); + RCU_QSBR_RETURN_IF_ERROR((ret == 1), "Non-blocking QSBR check"); + + for (i = 0; i < 3; i++) + rte_rcu_qsbr_unregister_thread(&t[0], enabled_core_ids[i]); + + ret = rte_rcu_qsbr_check(&t[0], token, true); + RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Blocking QSBR check"); + + return 0; +} + +/* + * rte_rcu_qsbr_dump: Dump status of a single QS variable to a file + */ +static int +test_rcu_qsbr_dump(void) +{ + int i; + + printf("\nTest rte_rcu_qsbr_dump()\n"); + + rte_rcu_qsbr_init(&t[0]); + rte_rcu_qsbr_init(&t[1]); + + /* QS variable with 0 core mask */ + rte_rcu_qsbr_dump(stdout, &t[0]); + + rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[0]); + + for (i = 1; i < 3; i++) + rte_rcu_qsbr_register_thread(&t[1], enabled_core_ids[i]); + + rte_rcu_qsbr_dump(stdout, &t[0]); + rte_rcu_qsbr_dump(stdout, &t[1]); + printf("\n"); + return 0; +} + +static int +test_rcu_qsbr_reader(void *arg) +{ + struct rte_rcu_qsbr *temp; + struct rte_hash *hash = NULL; + int i; + uint32_t lcore_id = rte_lcore_id(); + uint8_t read_type = (uint8_t)((uintptr_t)arg); + uint32_t *pdata; + + temp = &t[read_type]; + hash = h[read_type]; + + do { + rte_rcu_qsbr_register_thread(temp, lcore_id); + for (i = 0; i < TOTAL_ENTRY; i++) { + if (rte_hash_lookup_data(hash, keys+i, + (void **)&pdata) != -ENOENT) { + *pdata = 0; + while (*pdata < COUNTER_VALUE) + ++*pdata; + } + } + /* Update quiescent state counter */ + rte_rcu_qsbr_update(temp, lcore_id); + rte_rcu_qsbr_unregister_thread(temp, lcore_id); + } while (!writer_done); + + return 0; +} + +static int +test_rcu_qsbr_writer(void *arg) +{ + uint64_t token; + int32_t pos; + struct rte_rcu_qsbr *temp; + struct rte_hash *hash = NULL; + uint8_t writer_type = (uint8_t)((uintptr_t)arg); + + temp = &t[(writer_type/2) % RTE_RCU_MAX_LCORE]; + hash = h[(writer_type/2) % RTE_RCU_MAX_LCORE]; + + /* Delete element from the shared data structure */ + pos = rte_hash_del_key(hash, keys + (writer_type % TOTAL_ENTRY)); + if (pos < 0) { + printf("Delete key failed #%d\n", + keys[writer_type % TOTAL_ENTRY]); + return -1; + } + /* + * Start the quiescent state query process + * Note: Expected Quiescent states kept greater than 1 for test only + */ + rte_rcu_qsbr_start(temp, writer_type + 1, &token); + /* Check the quiescent state status */ + rte_rcu_qsbr_check(temp, token, true); + if (*hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE] + [writer_type % TOTAL_ENTRY] != COUNTER_VALUE && + *hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE] + [writer_type % TOTAL_ENTRY] != 0) { + printf("Reader did not complete #%d = %d\t", writer_type, + *hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE] + [writer_type % TOTAL_ENTRY]); + return -1; + } + + if (rte_hash_free_key_with_position(hash, pos) < 0) { + printf("Failed to free the key #%d\n", + keys[writer_type % TOTAL_ENTRY]); + return -1; + } + rte_free(hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE] + [writer_type % TOTAL_ENTRY]); + hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE] + [writer_type % TOTAL_ENTRY] = NULL; + + return 0; +} + +static struct rte_hash * +init_hash(int hash_id) +{ + int i; + struct rte_hash *h = NULL; + sprintf(hash_name[hash_id], "hash%d", hash_id); + struct rte_hash_parameters hash_params = { + .entries = TOTAL_ENTRY, + .key_len = sizeof(uint32_t), + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + .hash_func = rte_hash_crc, + .extra_flag = + RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF, + .name = hash_name[hash_id], + }; + + h = rte_hash_create(&hash_params); + if (h == NULL) { + printf("Hash create Failed\n"); + return NULL; + } + + for (i = 0; i < TOTAL_ENTRY; i++) { + hash_data[hash_id][i] = rte_zmalloc(NULL, sizeof(uint32_t), 0); + if (hash_data[hash_id][i] == NULL) { + printf("No memory\n"); + return NULL; + } + } + keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0); + if (keys == NULL) { + printf("No memory\n"); + return NULL; + } + + for (i = 0; i < TOTAL_ENTRY; i++) + keys[i] = i; + + for (i = 0; i < TOTAL_ENTRY; i++) { + if (rte_hash_add_key_data(h, keys + i, + (void *)((uintptr_t)hash_data[hash_id][i])) + < 0) { + printf("Hash key add Failed #%d\n", i); + return NULL; + } + } + return h; +} + +/* + * Functional test: + * Single writer, Single QS variable Single QSBR query, Blocking rcu_qsbr_check + */ +static int +test_rcu_qsbr_sw_sv_1qs(void) +{ + uint64_t token; + int i; + int32_t pos; + writer_done = 0; + + printf("\nTest: 1 writer, 1 QSBR variable, 1 QSBR Query, " + "Blocking QSBR Check\n"); + + /* QS variable is initialized */ + rte_rcu_qsbr_init(&t[0]); + + /* Register worker threads on 4 cores */ + for (i = 0; i < 4; i++) + rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]); + + /* Shared data structure created */ + h[0] = init_hash(0); + if (h[0] == NULL) { + printf("Hash init failed\n"); + goto error; + } + + /* Reader threads are launched */ + for (i = 0; i < 4; i++) + rte_eal_remote_launch(test_rcu_qsbr_reader, NULL, + enabled_core_ids[i]); + + for (i = 0; i < TOTAL_ENTRY; i++) { + /* Delete elements from the shared data structure */ + pos = rte_hash_del_key(h[0], keys + i); + if (pos < 0) { + printf("Delete key failed #%d\n", keys[i]); + goto error; + } + /* Start the quiescent state query process */ + rte_rcu_qsbr_start(&t[0], num_1qs, &token); + + /* Check the quiescent state status */ + rte_rcu_qsbr_check(&t[0], token, true); + if (*hash_data[0][i] != COUNTER_VALUE && + *hash_data[0][i] != 0) { + printf("Reader did not complete #%d = %d\n", i, + *hash_data[0][i]); + goto error; + } + + if (rte_hash_free_key_with_position(h[0], pos) < 0) { + printf("Failed to free the key #%d\n", keys[i]); + goto error; + } + rte_free(hash_data[0][i]); + hash_data[0][i] = NULL; + } + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + /* Check return value from threads */ + for (i = 0; i < 4; i++) + if (lcore_config[enabled_core_ids[i]].ret < 0) + goto error; + rte_hash_free(h[0]); + rte_free(keys); + + return 0; + +error: + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + + rte_hash_free(h[0]); + rte_free(keys); + for (i = 0; i < TOTAL_ENTRY; i++) + rte_free(hash_data[0][i]); + + return -1; +} + +/* + * Functional test: + * Single writer, Single QS variable, Single QSBR query, + * Non-blocking rcu_qsbr_check + */ +static int +test_rcu_qsbr_sw_sv_1qs_non_blocking(void) +{ + uint64_t token; + int i, ret; + int32_t pos; + writer_done = 0; + + printf("Test: 1 writer, 1 QSBR variable, 1 QSBR Query, " + "Non-Blocking QSBR check\n"); + + rte_rcu_qsbr_init(&t[0]); + /* Register worker threads on 4 cores */ + for (i = 0; i < 4; i++) + rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]); + + /* Shared data structure created */ + h[0] = init_hash(0); + if (h[0] == NULL) { + printf("Hash init failed\n"); + goto error; + } + + /* Reader threads are launched */ + for (i = 0; i < 4; i++) + rte_eal_remote_launch(test_rcu_qsbr_reader, NULL, + enabled_core_ids[i]); + + for (i = 0; i < TOTAL_ENTRY; i++) { + /* Delete elements from the shared data structure */ + pos = rte_hash_del_key(h[0], keys + i); + if (pos < 0) { + printf("Delete key failed #%d\n", keys[i]); + goto error; + } + /* Start the quiescent state query process */ + rte_rcu_qsbr_start(&t[0], num_1qs, &token); + + /* Check the quiescent state status */ + do { + ret = rte_rcu_qsbr_check(&t[0], token, false); + } while (ret == 0); + if (*hash_data[0][i] != COUNTER_VALUE && + *hash_data[0][i] != 0) { + printf("Reader did not complete #%d = %d\n", i, + *hash_data[0][i]); + goto error; + } + + if (rte_hash_free_key_with_position(h[0], pos) < 0) { + printf("Failed to free the key #%d\n", keys[i]); + goto error; + } + rte_free(hash_data[0][i]); + hash_data[0][i] = NULL; + } + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + /* Check return value from threads */ + for (i = 0; i < 4; i++) + if (lcore_config[enabled_core_ids[i]].ret < 0) + goto error; + rte_hash_free(h[0]); + rte_free(keys); + + return 0; + +error: + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + + rte_hash_free(h[0]); + rte_free(keys); + for (i = 0; i < TOTAL_ENTRY; i++) + rte_free(hash_data[0][i]); + + return -1; +} + +/* + * Functional test: + * Single writer, Single QS variable, simultaneous QSBR Queries + */ +static int +test_rcu_qsbr_sw_sv_3qs(void) +{ + uint64_t token[3]; + int i; + int32_t pos[3]; + writer_done = 0; + + printf("Test: 1 writer, 1 QSBR variable, simultaneous QSBR queries\n"); + + rte_rcu_qsbr_init(&t[0]); + + /* Register worker threads on 4 cores */ + for (i = 0; i < 4; i++) + rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]); + + /* Shared data structure created */ + h[0] = init_hash(0); + if (h[0] == NULL) { + printf("Hash init failed\n"); + goto error; + } + + /* Reader threads are launched */ + for (i = 0; i < 4; i++) + rte_eal_remote_launch(test_rcu_qsbr_reader, NULL, + enabled_core_ids[i]); + + /* Delete element from the shared data structure */ + pos[0] = rte_hash_del_key(h[0], keys + 0); + if (pos[0] < 0) { + printf("Delete key failed #%d\n", keys[0]); + goto error; + } + /* Start the quiescent state query process */ + rte_rcu_qsbr_start(&t[0], num_1qs, &token[0]); + + /* Delete element from the shared data structure */ + pos[1] = rte_hash_del_key(h[0], keys + 3); + if (pos[1] < 0) { + printf("Delete key failed #%d\n", keys[3]); + goto error; + } + /* + * Start the quiescent state query process + * Note: num_2qs kept greater than 1 for test only + */ + rte_rcu_qsbr_start(&t[0], num_2qs, &token[1]); + + /* Delete element from the shared data structure */ + pos[2] = rte_hash_del_key(h[0], keys + 6); + if (pos[2] < 0) { + printf("Delete key failed #%d\n", keys[6]); + goto error; + } + /* + * Start the quiescent state query process + * Note: num_3qs kept greater than 1 for test only + */ + rte_rcu_qsbr_start(&t[0], num_3qs, &token[2]); + + /* Check the quiescent state status */ + rte_rcu_qsbr_check(&t[0], token[0], true); + if (*hash_data[0][0] != COUNTER_VALUE && *hash_data[0][0] != 0) { + printf("Reader did not complete #0 = %d\n", *hash_data[0][0]); + goto error; + } + + if (rte_hash_free_key_with_position(h[0], pos[0]) < 0) { + printf("Failed to free the key #%d\n", keys[0]); + goto error; + } + rte_free(hash_data[0][0]); + hash_data[0][0] = NULL; + + /* Check the quiescent state status */ + rte_rcu_qsbr_check(&t[0], token[1], true); + if (*hash_data[0][3] != COUNTER_VALUE && *hash_data[0][3] != 0) { + printf("Reader did not complete #3 = %d\n", *hash_data[0][3]); + goto error; + } + + if (rte_hash_free_key_with_position(h[0], pos[1]) < 0) { + printf("Failed to free the key #%d\n", keys[3]); + goto error; + } + rte_free(hash_data[0][3]); + hash_data[0][3] = NULL; + + /* Check the quiescent state status */ + rte_rcu_qsbr_check(&t[0], token[2], true); + if (*hash_data[0][6] != COUNTER_VALUE && *hash_data[0][6] != 0) { + printf("Reader did not complete #6 = %d\n", *hash_data[0][6]); + goto error; + } + + if (rte_hash_free_key_with_position(h[0], pos[2]) < 0) { + printf("Failed to free the key #%d\n", keys[6]); + goto error; + } + rte_free(hash_data[0][6]); + hash_data[0][6] = NULL; + + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + /* Check return value from threads */ + for (i = 0; i < 4; i++) + if (lcore_config[enabled_core_ids[i]].ret < 0) + goto error; + rte_hash_free(h[0]); + rte_free(keys); + + return 0; + +error: + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + + rte_hash_free(h[0]); + rte_free(keys); + for (i = 0; i < TOTAL_ENTRY; i++) + rte_free(hash_data[0][i]); + + return -1; +} + +/* + * Multi writer, Multiple QS variable, simultaneous QSBR queries + */ +static int +test_rcu_qsbr_mw_mv_mqs(void) +{ + int i, j; + writer_done = 0; + uint8_t test_cores; + test_cores = num_cores / 4; + test_cores = test_cores * 4; + + printf("Test: %d writers, %d QSBR variable, Simultaneous QSBR queries\n" + , test_cores / 2, test_cores / 4); + + for (i = 0; i < num_cores / 4; i++) { + rte_rcu_qsbr_init(&t[i]); + h[i] = init_hash(i); + if (h[i] == NULL) { + printf("Hash init failed\n"); + goto error; + } + } + + /* Register worker threads on 2 cores */ + for (i = 0; i < test_cores / 2; i += 2) { + rte_rcu_qsbr_register_thread(&t[i / 2], enabled_core_ids[i]); + rte_rcu_qsbr_register_thread(&t[i / 2], + enabled_core_ids[i + 1]); + } + + /* Reader threads are launched */ + for (i = 0; i < test_cores / 2; i++) + rte_eal_remote_launch(test_rcu_qsbr_reader, + (void *)(uintptr_t)(i / 2), + enabled_core_ids[i]); + + /* Writer threads are launched */ + for (; i < test_cores; i++) + rte_eal_remote_launch(test_rcu_qsbr_writer, + (void *)(uintptr_t)(i - (test_cores / 2)), + enabled_core_ids[i]); + /* Wait for writers to complete */ + for (i = test_cores / 2; i < test_cores; i++) + rte_eal_wait_lcore(enabled_core_ids[i]); + + writer_done = 1; + /* Wait for readers to complete */ + rte_eal_mp_wait_lcore(); + + /* Check return value from threads */ + for (i = 0; i < test_cores; i++) + if (lcore_config[enabled_core_ids[i]].ret < 0) + goto error; + + for (i = 0; i < num_cores / 4; i++) + rte_hash_free(h[i]); + + rte_free(keys); + + return 0; + +error: + writer_done = 1; + /* Wait until all readers have exited */ + rte_eal_mp_wait_lcore(); + + for (i = 0; i < num_cores / 4; i++) + rte_hash_free(h[i]); + rte_free(keys); + for (j = 0; j < RTE_RCU_MAX_LCORE; j++) + for (i = 0; i < TOTAL_ENTRY; i++) + rte_free(hash_data[j][i]); + + return -1; +} + +static int +test_rcu_qsbr_main(void) +{ + if (get_enabled_cores_mask() != 0) + return -1; + + /* Error-checking test cases */ + if (test_rcu_qsbr_register_thread() < 0) + goto test_fail; + + if (test_rcu_qsbr_unregister_thread() < 0) + goto test_fail; + + if (test_rcu_qsbr_start() < 0) + goto test_fail; + + if (test_rcu_qsbr_check() < 0) + goto test_fail; + + if (test_rcu_qsbr_dump() < 0) + goto test_fail; + + /* Functional test cases */ + if (num_cores < 4) { + printf("Test failed! Need 4 or more cores\n"); + goto test_fail; + } + if (test_rcu_qsbr_sw_sv_1qs() < 0) + goto test_fail; + + if (test_rcu_qsbr_sw_sv_1qs_non_blocking() < 0) + goto test_fail; + + if (test_rcu_qsbr_sw_sv_3qs() < 0) + goto test_fail; + + if (test_rcu_qsbr_mw_mv_mqs() < 0) + goto test_fail; + + printf("\n"); + return 0; + +test_fail: + return -1; +} + +REGISTER_TEST_COMMAND(rcu_qsbr_autotest, test_rcu_qsbr_main);