From patchwork Fri Aug 11 17:32:12 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Tyler Retzlaff X-Patchwork-Id: 130183 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id DE04E43036; Fri, 11 Aug 2023 19:32:39 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 148234326B; Fri, 11 Aug 2023 19:32:24 +0200 (CEST) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mails.dpdk.org (Postfix) with ESMTP id B9CAC40F16; Fri, 11 Aug 2023 19:32:19 +0200 (CEST) Received: by linux.microsoft.com (Postfix, from userid 1086) id 0D91E20FCF79; Fri, 11 Aug 2023 10:32:18 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 0D91E20FCF79 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1691775139; bh=H3TS+tL24xq5jdqtCcsVt9QXPHAQ77wp1K+DvFPNrIY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=G3AoVilnTZ1jtwnY0znhNou+QudWAT9FcBUdfJtto50+FrQ2f/uMuXTRR7FY3PKXE gs0XUqm7dP55oSVbNnIEu9YWAyt9RZNlUeYyZL3UUWo/qsWafbNwSrQDEduT2Zu+Rc xR6ex0KkeMTebF3T4zaPY31WYIqqX2TwWNW2xgK0= From: Tyler Retzlaff To: dev@dpdk.org Cc: techboard@dpdk.org, Bruce Richardson , Honnappa Nagarahalli , Ruifeng Wang , Jerin Jacob , Sunil Kumar Kori , =?utf-8?q?Mattias_R=C3=B6nnblom?= , Joyce Kong , David Christensen , Konstantin Ananyev , David Hunt , Thomas Monjalon , David Marchand , Tyler Retzlaff Subject: [PATCH v2 2/6] eal: adapt EAL to present rte optional atomics API Date: Fri, 11 Aug 2023 10:32:12 -0700 Message-Id: <1691775136-6460-3-git-send-email-roretzla@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1691775136-6460-1-git-send-email-roretzla@linux.microsoft.com> References: <1691717521-1025-1-git-send-email-roretzla@linux.microsoft.com> <1691775136-6460-1-git-send-email-roretzla@linux.microsoft.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Adapt the EAL public headers to use rte optional atomics API instead of directly using and exposing toolchain specific atomic builtin intrinsics. Signed-off-by: Tyler Retzlaff Reviewed-by: Morten Brørup --- app/test/test_mcslock.c | 6 ++-- lib/eal/arm/include/rte_atomic_32.h | 4 +-- lib/eal/arm/include/rte_atomic_64.h | 36 +++++++++++------------ lib/eal/arm/include/rte_pause_64.h | 26 ++++++++-------- lib/eal/arm/rte_power_intrinsics.c | 8 ++--- lib/eal/common/eal_common_trace.c | 16 +++++----- lib/eal/include/generic/rte_atomic.h | 50 +++++++++++++++---------------- lib/eal/include/generic/rte_pause.h | 38 ++++++++++++------------ lib/eal/include/generic/rte_rwlock.h | 47 +++++++++++++++-------------- lib/eal/include/generic/rte_spinlock.h | 19 ++++++------ lib/eal/include/rte_mcslock.h | 50 +++++++++++++++---------------- lib/eal/include/rte_pflock.h | 24 ++++++++------- lib/eal/include/rte_seqcount.h | 18 ++++++------ lib/eal/include/rte_ticketlock.h | 42 +++++++++++++------------- lib/eal/include/rte_trace_point.h | 4 +-- lib/eal/loongarch/include/rte_atomic.h | 4 +-- lib/eal/ppc/include/rte_atomic.h | 54 +++++++++++++++++----------------- lib/eal/riscv/include/rte_atomic.h | 4 +-- lib/eal/x86/include/rte_atomic.h | 8 ++--- lib/eal/x86/include/rte_spinlock.h | 2 +- lib/eal/x86/rte_power_intrinsics.c | 6 ++-- 21 files changed, 237 insertions(+), 229 deletions(-) diff --git a/app/test/test_mcslock.c b/app/test/test_mcslock.c index 52e45e7..cc25970 100644 --- a/app/test/test_mcslock.c +++ b/app/test/test_mcslock.c @@ -36,9 +36,9 @@ * lock multiple times. */ -rte_mcslock_t *p_ml; -rte_mcslock_t *p_ml_try; -rte_mcslock_t *p_ml_perf; +rte_mcslock_t * __rte_atomic p_ml; +rte_mcslock_t * __rte_atomic p_ml_try; +rte_mcslock_t * __rte_atomic p_ml_perf; static unsigned int count; diff --git a/lib/eal/arm/include/rte_atomic_32.h b/lib/eal/arm/include/rte_atomic_32.h index c00ab78..62fc337 100644 --- a/lib/eal/arm/include/rte_atomic_32.h +++ b/lib/eal/arm/include/rte_atomic_32.h @@ -34,9 +34,9 @@ #define rte_io_rmb() rte_rmb() static __rte_always_inline void -rte_atomic_thread_fence(int memorder) +rte_atomic_thread_fence(rte_memory_order memorder) { - __atomic_thread_fence(memorder); + __rte_atomic_thread_fence(memorder); } #ifdef __cplusplus diff --git a/lib/eal/arm/include/rte_atomic_64.h b/lib/eal/arm/include/rte_atomic_64.h index 6047911..75d8ba6 100644 --- a/lib/eal/arm/include/rte_atomic_64.h +++ b/lib/eal/arm/include/rte_atomic_64.h @@ -38,9 +38,9 @@ #define rte_io_rmb() rte_rmb() static __rte_always_inline void -rte_atomic_thread_fence(int memorder) +rte_atomic_thread_fence(rte_memory_order memorder) { - __atomic_thread_fence(memorder); + __rte_atomic_thread_fence(memorder); } /*------------------------ 128 bit atomic operations -------------------------*/ @@ -107,33 +107,33 @@ */ RTE_SET_USED(failure); /* Find invalid memory order */ - RTE_ASSERT(success == __ATOMIC_RELAXED || - success == __ATOMIC_ACQUIRE || - success == __ATOMIC_RELEASE || - success == __ATOMIC_ACQ_REL || - success == __ATOMIC_SEQ_CST); + RTE_ASSERT(success == rte_memory_order_relaxed || + success == rte_memory_order_acquire || + success == rte_memory_order_release || + success == rte_memory_order_acq_rel || + success == rte_memory_order_seq_cst); rte_int128_t expected = *exp; rte_int128_t desired = *src; rte_int128_t old; #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) - if (success == __ATOMIC_RELAXED) + if (success == rte_memory_order_relaxed) __cas_128_relaxed(dst, exp, desired); - else if (success == __ATOMIC_ACQUIRE) + else if (success == rte_memory_order_acquire) __cas_128_acquire(dst, exp, desired); - else if (success == __ATOMIC_RELEASE) + else if (success == rte_memory_order_release) __cas_128_release(dst, exp, desired); else __cas_128_acq_rel(dst, exp, desired); old = *exp; #else -#define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) -#define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \ - (mo) == __ATOMIC_SEQ_CST) +#define __HAS_ACQ(mo) ((mo) != rte_memory_order_relaxed && (mo) != rte_memory_order_release) +#define __HAS_RLS(mo) ((mo) == rte_memory_order_release || (mo) == rte_memory_order_acq_rel || \ + (mo) == rte_memory_order_seq_cst) - int ldx_mo = __HAS_ACQ(success) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED; - int stx_mo = __HAS_RLS(success) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED; + int ldx_mo = __HAS_ACQ(success) ? rte_memory_order_acquire : rte_memory_order_relaxed; + int stx_mo = __HAS_RLS(success) ? rte_memory_order_release : rte_memory_order_relaxed; #undef __HAS_ACQ #undef __HAS_RLS @@ -153,7 +153,7 @@ : "Q" (src->val[0]) \ : "memory"); } - if (ldx_mo == __ATOMIC_RELAXED) + if (ldx_mo == rte_memory_order_relaxed) __LOAD_128("ldxp", dst, old) else __LOAD_128("ldaxp", dst, old) @@ -170,7 +170,7 @@ : "memory"); } if (likely(old.int128 == expected.int128)) { - if (stx_mo == __ATOMIC_RELAXED) + if (stx_mo == rte_memory_order_relaxed) __STORE_128("stxp", dst, desired, ret) else __STORE_128("stlxp", dst, desired, ret) @@ -181,7 +181,7 @@ * needs to be stored back to ensure it was read * atomically. */ - if (stx_mo == __ATOMIC_RELAXED) + if (stx_mo == rte_memory_order_relaxed) __STORE_128("stxp", dst, old, ret) else __STORE_128("stlxp", dst, old, ret) diff --git a/lib/eal/arm/include/rte_pause_64.h b/lib/eal/arm/include/rte_pause_64.h index 5f70e97..d4daafc 100644 --- a/lib/eal/arm/include/rte_pause_64.h +++ b/lib/eal/arm/include/rte_pause_64.h @@ -41,7 +41,7 @@ static inline void rte_pause(void) * implicitly to exit WFE. */ #define __RTE_ARM_LOAD_EXC_8(src, dst, memorder) { \ - if (memorder == __ATOMIC_RELAXED) { \ + if (memorder == rte_memory_order_relaxed) { \ asm volatile("ldxrb %w[tmp], [%x[addr]]" \ : [tmp] "=&r" (dst) \ : [addr] "r" (src) \ @@ -60,7 +60,7 @@ static inline void rte_pause(void) * implicitly to exit WFE. */ #define __RTE_ARM_LOAD_EXC_16(src, dst, memorder) { \ - if (memorder == __ATOMIC_RELAXED) { \ + if (memorder == rte_memory_order_relaxed) { \ asm volatile("ldxrh %w[tmp], [%x[addr]]" \ : [tmp] "=&r" (dst) \ : [addr] "r" (src) \ @@ -79,7 +79,7 @@ static inline void rte_pause(void) * implicitly to exit WFE. */ #define __RTE_ARM_LOAD_EXC_32(src, dst, memorder) { \ - if (memorder == __ATOMIC_RELAXED) { \ + if (memorder == rte_memory_order_relaxed) { \ asm volatile("ldxr %w[tmp], [%x[addr]]" \ : [tmp] "=&r" (dst) \ : [addr] "r" (src) \ @@ -98,7 +98,7 @@ static inline void rte_pause(void) * implicitly to exit WFE. */ #define __RTE_ARM_LOAD_EXC_64(src, dst, memorder) { \ - if (memorder == __ATOMIC_RELAXED) { \ + if (memorder == rte_memory_order_relaxed) { \ asm volatile("ldxr %x[tmp], [%x[addr]]" \ : [tmp] "=&r" (dst) \ : [addr] "r" (src) \ @@ -118,7 +118,7 @@ static inline void rte_pause(void) */ #define __RTE_ARM_LOAD_EXC_128(src, dst, memorder) { \ volatile rte_int128_t *dst_128 = (volatile rte_int128_t *)&dst; \ - if (memorder == __ATOMIC_RELAXED) { \ + if (memorder == rte_memory_order_relaxed) { \ asm volatile("ldxp %x[tmp0], %x[tmp1], [%x[addr]]" \ : [tmp0] "=&r" (dst_128->val[0]), \ [tmp1] "=&r" (dst_128->val[1]) \ @@ -153,8 +153,8 @@ static inline void rte_pause(void) { uint16_t value; - RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && - memorder != __ATOMIC_RELAXED); + RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire && + memorder != rte_memory_order_relaxed); __RTE_ARM_LOAD_EXC_16(addr, value, memorder) if (value != expected) { @@ -172,8 +172,8 @@ static inline void rte_pause(void) { uint32_t value; - RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && - memorder != __ATOMIC_RELAXED); + RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire && + memorder != rte_memory_order_relaxed); __RTE_ARM_LOAD_EXC_32(addr, value, memorder) if (value != expected) { @@ -191,8 +191,8 @@ static inline void rte_pause(void) { uint64_t value; - RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && - memorder != __ATOMIC_RELAXED); + RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire && + memorder != rte_memory_order_relaxed); __RTE_ARM_LOAD_EXC_64(addr, value, memorder) if (value != expected) { @@ -206,8 +206,8 @@ static inline void rte_pause(void) #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \ RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \ - RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && \ - memorder != __ATOMIC_RELAXED); \ + RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire && \ + memorder != rte_memory_order_relaxed); \ const uint32_t size = sizeof(*(addr)) << 3; \ typeof(*(addr)) expected_value = (expected); \ typeof(*(addr)) value; \ diff --git a/lib/eal/arm/rte_power_intrinsics.c b/lib/eal/arm/rte_power_intrinsics.c index 77b96e4..f54cf59 100644 --- a/lib/eal/arm/rte_power_intrinsics.c +++ b/lib/eal/arm/rte_power_intrinsics.c @@ -33,19 +33,19 @@ switch (pmc->size) { case sizeof(uint8_t): - __RTE_ARM_LOAD_EXC_8(pmc->addr, cur_value, __ATOMIC_RELAXED) + __RTE_ARM_LOAD_EXC_8(pmc->addr, cur_value, rte_memory_order_relaxed) __RTE_ARM_WFE() break; case sizeof(uint16_t): - __RTE_ARM_LOAD_EXC_16(pmc->addr, cur_value, __ATOMIC_RELAXED) + __RTE_ARM_LOAD_EXC_16(pmc->addr, cur_value, rte_memory_order_relaxed) __RTE_ARM_WFE() break; case sizeof(uint32_t): - __RTE_ARM_LOAD_EXC_32(pmc->addr, cur_value, __ATOMIC_RELAXED) + __RTE_ARM_LOAD_EXC_32(pmc->addr, cur_value, rte_memory_order_relaxed) __RTE_ARM_WFE() break; case sizeof(uint64_t): - __RTE_ARM_LOAD_EXC_64(pmc->addr, cur_value, __ATOMIC_RELAXED) + __RTE_ARM_LOAD_EXC_64(pmc->addr, cur_value, rte_memory_order_relaxed) __RTE_ARM_WFE() break; default: diff --git a/lib/eal/common/eal_common_trace.c b/lib/eal/common/eal_common_trace.c index cb980af..c6628dd 100644 --- a/lib/eal/common/eal_common_trace.c +++ b/lib/eal/common/eal_common_trace.c @@ -103,11 +103,11 @@ struct trace_point_head * trace_mode_set(rte_trace_point_t *t, enum rte_trace_mode mode) { if (mode == RTE_TRACE_MODE_OVERWRITE) - __atomic_fetch_and(t, ~__RTE_TRACE_FIELD_ENABLE_DISCARD, - __ATOMIC_RELEASE); + rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_DISCARD, + rte_memory_order_release); else - __atomic_fetch_or(t, __RTE_TRACE_FIELD_ENABLE_DISCARD, - __ATOMIC_RELEASE); + rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_DISCARD, + rte_memory_order_release); } void @@ -141,7 +141,7 @@ rte_trace_mode rte_trace_mode_get(void) if (trace_point_is_invalid(t)) return false; - val = __atomic_load_n(t, __ATOMIC_ACQUIRE); + val = rte_atomic_load_explicit(t, rte_memory_order_acquire); return (val & __RTE_TRACE_FIELD_ENABLE_MASK) != 0; } @@ -153,7 +153,8 @@ rte_trace_mode rte_trace_mode_get(void) if (trace_point_is_invalid(t)) return -ERANGE; - prev = __atomic_fetch_or(t, __RTE_TRACE_FIELD_ENABLE_MASK, __ATOMIC_RELEASE); + prev = rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_MASK, + rte_memory_order_release); if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) == 0) __atomic_fetch_add(&trace.status, 1, __ATOMIC_RELEASE); return 0; @@ -167,7 +168,8 @@ rte_trace_mode rte_trace_mode_get(void) if (trace_point_is_invalid(t)) return -ERANGE; - prev = __atomic_fetch_and(t, ~__RTE_TRACE_FIELD_ENABLE_MASK, __ATOMIC_RELEASE); + prev = rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_MASK, + rte_memory_order_release); if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) != 0) __atomic_fetch_sub(&trace.status, 1, __ATOMIC_RELEASE); return 0; diff --git a/lib/eal/include/generic/rte_atomic.h b/lib/eal/include/generic/rte_atomic.h index efd29eb..f6c4b3e 100644 --- a/lib/eal/include/generic/rte_atomic.h +++ b/lib/eal/include/generic/rte_atomic.h @@ -63,7 +63,7 @@ * but has different syntax and memory ordering semantic. Hence * deprecated for the simplicity of memory ordering semantics in use. * - * rte_atomic_thread_fence(__ATOMIC_ACQ_REL) should be used instead. + * rte_atomic_thread_fence(rte_memory_order_acq_rel) should be used instead. */ static inline void rte_smp_mb(void); @@ -80,7 +80,7 @@ * but has different syntax and memory ordering semantic. Hence * deprecated for the simplicity of memory ordering semantics in use. * - * rte_atomic_thread_fence(__ATOMIC_RELEASE) should be used instead. + * rte_atomic_thread_fence(rte_memory_order_release) should be used instead. * The fence also guarantees LOAD operations that precede the call * are globally visible across the lcores before the STORE operations * that follows it. @@ -100,7 +100,7 @@ * but has different syntax and memory ordering semantic. Hence * deprecated for the simplicity of memory ordering semantics in use. * - * rte_atomic_thread_fence(__ATOMIC_ACQUIRE) should be used instead. + * rte_atomic_thread_fence(rte_memory_order_acquire) should be used instead. * The fence also guarantees LOAD operations that precede the call * are globally visible across the lcores before the STORE operations * that follows it. @@ -154,7 +154,7 @@ /** * Synchronization fence between threads based on the specified memory order. */ -static inline void rte_atomic_thread_fence(int memorder); +static inline void rte_atomic_thread_fence(rte_memory_order memorder); /*------------------------- 16 bit atomic operations -------------------------*/ @@ -207,7 +207,7 @@ static inline uint16_t rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val) { - return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST); + return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst); } #endif @@ -274,7 +274,7 @@ static inline void rte_atomic16_add(rte_atomic16_t *v, int16_t inc) { - __atomic_fetch_add(&v->cnt, inc, __ATOMIC_SEQ_CST); + rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst); } /** @@ -288,7 +288,7 @@ static inline void rte_atomic16_sub(rte_atomic16_t *v, int16_t dec) { - __atomic_fetch_sub(&v->cnt, dec, __ATOMIC_SEQ_CST); + rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst); } /** @@ -341,7 +341,7 @@ static inline int16_t rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc) { - return __atomic_fetch_add(&v->cnt, inc, __ATOMIC_SEQ_CST) + inc; + return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc; } /** @@ -361,7 +361,7 @@ static inline int16_t rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec) { - return __atomic_fetch_sub(&v->cnt, dec, __ATOMIC_SEQ_CST) - dec; + return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec; } /** @@ -380,7 +380,7 @@ #ifdef RTE_FORCE_INTRINSICS static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v) { - return __atomic_fetch_add(&v->cnt, 1, __ATOMIC_SEQ_CST) + 1 == 0; + return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_seq_cst) + 1 == 0; } #endif @@ -400,7 +400,7 @@ static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v) #ifdef RTE_FORCE_INTRINSICS static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) { - return __atomic_fetch_sub(&v->cnt, 1, __ATOMIC_SEQ_CST) - 1 == 0; + return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_seq_cst) - 1 == 0; } #endif @@ -486,7 +486,7 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v) static inline uint32_t rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val) { - return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST); + return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst); } #endif @@ -553,7 +553,7 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v) static inline void rte_atomic32_add(rte_atomic32_t *v, int32_t inc) { - __atomic_fetch_add(&v->cnt, inc, __ATOMIC_SEQ_CST); + rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst); } /** @@ -567,7 +567,7 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v) static inline void rte_atomic32_sub(rte_atomic32_t *v, int32_t dec) { - __atomic_fetch_sub(&v->cnt, dec, __ATOMIC_SEQ_CST); + rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst); } /** @@ -620,7 +620,7 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v) static inline int32_t rte_atomic32_add_return(rte_atomic32_t *v, int32_t inc) { - return __atomic_fetch_add(&v->cnt, inc, __ATOMIC_SEQ_CST) + inc; + return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc; } /** @@ -640,7 +640,7 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v) static inline int32_t rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec) { - return __atomic_fetch_sub(&v->cnt, dec, __ATOMIC_SEQ_CST) - dec; + return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec; } /** @@ -659,7 +659,7 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v) #ifdef RTE_FORCE_INTRINSICS static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v) { - return __atomic_fetch_add(&v->cnt, 1, __ATOMIC_SEQ_CST) + 1 == 0; + return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_seq_cst) + 1 == 0; } #endif @@ -679,7 +679,7 @@ static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v) #ifdef RTE_FORCE_INTRINSICS static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) { - return __atomic_fetch_sub(&v->cnt, 1, __ATOMIC_SEQ_CST) - 1 == 0; + return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_seq_cst) - 1 == 0; } #endif @@ -764,7 +764,7 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v) static inline uint64_t rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val) { - return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST); + return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst); } #endif @@ -885,7 +885,7 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v) static inline void rte_atomic64_add(rte_atomic64_t *v, int64_t inc) { - __atomic_fetch_add(&v->cnt, inc, __ATOMIC_SEQ_CST); + rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst); } #endif @@ -904,7 +904,7 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v) static inline void rte_atomic64_sub(rte_atomic64_t *v, int64_t dec) { - __atomic_fetch_sub(&v->cnt, dec, __ATOMIC_SEQ_CST); + rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst); } #endif @@ -962,7 +962,7 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v) static inline int64_t rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc) { - return __atomic_fetch_add(&v->cnt, inc, __ATOMIC_SEQ_CST) + inc; + return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc; } #endif @@ -986,7 +986,7 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v) static inline int64_t rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec) { - return __atomic_fetch_sub(&v->cnt, dec, __ATOMIC_SEQ_CST) - dec; + return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec; } #endif @@ -1117,8 +1117,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v) * stronger) model. * @param failure * If unsuccessful, the operation's memory behavior conforms to this (or a - * stronger) model. This argument cannot be __ATOMIC_RELEASE, - * __ATOMIC_ACQ_REL, or a stronger model than success. + * stronger) model. This argument cannot be rte_memory_order_release, + * rte_memory_order_acq_rel, or a stronger model than success. * @return * Non-zero on success; 0 on failure. */ diff --git a/lib/eal/include/generic/rte_pause.h b/lib/eal/include/generic/rte_pause.h index bebfa95..c816e7d 100644 --- a/lib/eal/include/generic/rte_pause.h +++ b/lib/eal/include/generic/rte_pause.h @@ -36,13 +36,13 @@ * A 16-bit expected value to be in the memory location. * @param memorder * Two different memory orders that can be specified: - * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * rte_memory_order_acquire and rte_memory_order_relaxed. These map to * C++11 memory orders with the same names, see the C++11 standard or * the GCC wiki on atomic synchronization for detailed definition. */ static __rte_always_inline void rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, - int memorder); + rte_memory_order memorder); /** * Wait for *addr to be updated with a 32-bit expected value, with a relaxed @@ -54,13 +54,13 @@ * A 32-bit expected value to be in the memory location. * @param memorder * Two different memory orders that can be specified: - * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * rte_memory_order_acquire and rte_memory_order_relaxed. These map to * C++11 memory orders with the same names, see the C++11 standard or * the GCC wiki on atomic synchronization for detailed definition. */ static __rte_always_inline void rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, - int memorder); + rte_memory_order memorder); /** * Wait for *addr to be updated with a 64-bit expected value, with a relaxed @@ -72,42 +72,42 @@ * A 64-bit expected value to be in the memory location. * @param memorder * Two different memory orders that can be specified: - * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * rte_memory_order_acquire and rte_memory_order_relaxed. These map to * C++11 memory orders with the same names, see the C++11 standard or * the GCC wiki on atomic synchronization for detailed definition. */ static __rte_always_inline void rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, - int memorder); + rte_memory_order memorder); #ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED static __rte_always_inline void rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, - int memorder) + rte_memory_order memorder) { - assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed); - while (__atomic_load_n(addr, memorder) != expected) + while (rte_atomic_load_explicit(addr, memorder) != expected) rte_pause(); } static __rte_always_inline void rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, - int memorder) + rte_memory_order memorder) { - assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed); - while (__atomic_load_n(addr, memorder) != expected) + while (rte_atomic_load_explicit(addr, memorder) != expected) rte_pause(); } static __rte_always_inline void rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, - int memorder) + rte_memory_order memorder) { - assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed); - while (__atomic_load_n(addr, memorder) != expected) + while (rte_atomic_load_explicit(addr, memorder) != expected) rte_pause(); } @@ -125,16 +125,16 @@ * An expected value to be in the memory location. * @param memorder * Two different memory orders that can be specified: - * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * rte_memory_order_acquire and rte_memory_order_relaxed. These map to * C++11 memory orders with the same names, see the C++11 standard or * the GCC wiki on atomic synchronization for detailed definition. */ #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \ RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \ - RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && \ - memorder != __ATOMIC_RELAXED); \ + RTE_BUILD_BUG_ON((memorder) != rte_memory_order_acquire && \ + (memorder) != rte_memory_order_relaxed); \ typeof(*(addr)) expected_value = (expected); \ - while (!((__atomic_load_n((addr), (memorder)) & (mask)) cond \ + while (!((rte_atomic_load_explicit((addr), (memorder)) & (mask)) cond \ expected_value)) \ rte_pause(); \ } while (0) diff --git a/lib/eal/include/generic/rte_rwlock.h b/lib/eal/include/generic/rte_rwlock.h index 24ebec6..176775f 100644 --- a/lib/eal/include/generic/rte_rwlock.h +++ b/lib/eal/include/generic/rte_rwlock.h @@ -58,7 +58,7 @@ #define RTE_RWLOCK_READ 0x4 /* Reader increment */ typedef struct __rte_lockable { - int32_t cnt; + int32_t __rte_atomic cnt; } rte_rwlock_t; /** @@ -93,21 +93,21 @@ while (1) { /* Wait while writer is present or pending */ - while (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED) + while (rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed) & RTE_RWLOCK_MASK) rte_pause(); /* Try to get read lock */ - x = __atomic_fetch_add(&rwl->cnt, RTE_RWLOCK_READ, - __ATOMIC_ACQUIRE) + RTE_RWLOCK_READ; + x = rte_atomic_fetch_add_explicit(&rwl->cnt, RTE_RWLOCK_READ, + rte_memory_order_acquire) + RTE_RWLOCK_READ; /* If no writer, then acquire was successful */ if (likely(!(x & RTE_RWLOCK_MASK))) return; /* Lost race with writer, backout the change. */ - __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ, - __ATOMIC_RELAXED); + rte_atomic_fetch_sub_explicit(&rwl->cnt, RTE_RWLOCK_READ, + rte_memory_order_relaxed); } } @@ -128,20 +128,20 @@ { int32_t x; - x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED); + x = rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed); /* fail if write lock is held or writer is pending */ if (x & RTE_RWLOCK_MASK) return -EBUSY; /* Try to get read lock */ - x = __atomic_fetch_add(&rwl->cnt, RTE_RWLOCK_READ, - __ATOMIC_ACQUIRE) + RTE_RWLOCK_READ; + x = rte_atomic_fetch_add_explicit(&rwl->cnt, RTE_RWLOCK_READ, + rte_memory_order_acquire) + RTE_RWLOCK_READ; /* Back out if writer raced in */ if (unlikely(x & RTE_RWLOCK_MASK)) { - __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ, - __ATOMIC_RELEASE); + rte_atomic_fetch_sub_explicit(&rwl->cnt, RTE_RWLOCK_READ, + rte_memory_order_release); return -EBUSY; } @@ -159,7 +159,7 @@ __rte_unlock_function(rwl) __rte_no_thread_safety_analysis { - __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_READ, __ATOMIC_RELEASE); + rte_atomic_fetch_sub_explicit(&rwl->cnt, RTE_RWLOCK_READ, rte_memory_order_release); } /** @@ -179,10 +179,10 @@ { int32_t x; - x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED); + x = rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed); if (x < RTE_RWLOCK_WRITE && - __atomic_compare_exchange_n(&rwl->cnt, &x, x + RTE_RWLOCK_WRITE, - 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) + rte_atomic_compare_exchange_weak_explicit(&rwl->cnt, &x, x + RTE_RWLOCK_WRITE, + rte_memory_order_acquire, rte_memory_order_relaxed)) return 0; else return -EBUSY; @@ -202,22 +202,25 @@ int32_t x; while (1) { - x = __atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED); + x = rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed); /* No readers or writers? */ if (likely(x < RTE_RWLOCK_WRITE)) { /* Turn off RTE_RWLOCK_WAIT, turn on RTE_RWLOCK_WRITE */ - if (__atomic_compare_exchange_n(&rwl->cnt, &x, RTE_RWLOCK_WRITE, 1, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) + if (rte_atomic_compare_exchange_weak_explicit( + &rwl->cnt, &x, RTE_RWLOCK_WRITE, + rte_memory_order_acquire, rte_memory_order_relaxed)) return; } /* Turn on writer wait bit */ if (!(x & RTE_RWLOCK_WAIT)) - __atomic_fetch_or(&rwl->cnt, RTE_RWLOCK_WAIT, __ATOMIC_RELAXED); + rte_atomic_fetch_or_explicit(&rwl->cnt, RTE_RWLOCK_WAIT, + rte_memory_order_relaxed); /* Wait until no readers before trying again */ - while (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED) > RTE_RWLOCK_WAIT) + while (rte_atomic_load_explicit(&rwl->cnt, + rte_memory_order_relaxed) > RTE_RWLOCK_WAIT) rte_pause(); } @@ -234,7 +237,7 @@ __rte_unlock_function(rwl) __rte_no_thread_safety_analysis { - __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_WRITE, __ATOMIC_RELEASE); + rte_atomic_fetch_sub_explicit(&rwl->cnt, RTE_RWLOCK_WRITE, rte_memory_order_release); } /** @@ -248,7 +251,7 @@ static inline int rte_rwlock_write_is_locked(rte_rwlock_t *rwl) { - if (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED) & RTE_RWLOCK_WRITE) + if (rte_atomic_load_explicit(&rwl->cnt, rte_memory_order_relaxed) & RTE_RWLOCK_WRITE) return 1; return 0; diff --git a/lib/eal/include/generic/rte_spinlock.h b/lib/eal/include/generic/rte_spinlock.h index e18f0cd..274616a 100644 --- a/lib/eal/include/generic/rte_spinlock.h +++ b/lib/eal/include/generic/rte_spinlock.h @@ -29,7 +29,7 @@ * The rte_spinlock_t type. */ typedef struct __rte_lockable { - volatile int locked; /**< lock status 0 = unlocked, 1 = locked */ + volatile int __rte_atomic locked; /**< lock status 0 = unlocked, 1 = locked */ } rte_spinlock_t; /** @@ -66,10 +66,10 @@ { int exp = 0; - while (!__atomic_compare_exchange_n(&sl->locked, &exp, 1, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { - rte_wait_until_equal_32((volatile uint32_t *)&sl->locked, - 0, __ATOMIC_RELAXED); + while (!rte_atomic_compare_exchange_strong_explicit(&sl->locked, &exp, 1, + rte_memory_order_acquire, rte_memory_order_relaxed)) { + rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&sl->locked, + 0, rte_memory_order_relaxed); exp = 0; } } @@ -90,7 +90,7 @@ rte_spinlock_unlock(rte_spinlock_t *sl) __rte_no_thread_safety_analysis { - __atomic_store_n(&sl->locked, 0, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&sl->locked, 0, rte_memory_order_release); } #endif @@ -113,9 +113,8 @@ __rte_no_thread_safety_analysis { int exp = 0; - return __atomic_compare_exchange_n(&sl->locked, &exp, 1, - 0, /* disallow spurious failure */ - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + return rte_atomic_compare_exchange_strong_explicit(&sl->locked, &exp, 1, + rte_memory_order_acquire, rte_memory_order_relaxed); } #endif @@ -129,7 +128,7 @@ */ static inline int rte_spinlock_is_locked (rte_spinlock_t *sl) { - return __atomic_load_n(&sl->locked, __ATOMIC_ACQUIRE); + return rte_atomic_load_explicit(&sl->locked, rte_memory_order_acquire); } /** diff --git a/lib/eal/include/rte_mcslock.h b/lib/eal/include/rte_mcslock.h index 18e63eb..229c8e2 100644 --- a/lib/eal/include/rte_mcslock.h +++ b/lib/eal/include/rte_mcslock.h @@ -33,8 +33,8 @@ * The rte_mcslock_t type. */ typedef struct rte_mcslock { - struct rte_mcslock *next; - int locked; /* 1 if the queue locked, 0 otherwise */ + struct rte_mcslock * __rte_atomic next; + int __rte_atomic locked; /* 1 if the queue locked, 0 otherwise */ } rte_mcslock_t; /** @@ -49,13 +49,13 @@ * lock should use its 'own node'. */ static inline void -rte_mcslock_lock(rte_mcslock_t **msl, rte_mcslock_t *me) +rte_mcslock_lock(rte_mcslock_t * __rte_atomic *msl, rte_mcslock_t *me) { rte_mcslock_t *prev; /* Init me node */ - __atomic_store_n(&me->locked, 1, __ATOMIC_RELAXED); - __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&me->locked, 1, rte_memory_order_relaxed); + rte_atomic_store_explicit(&me->next, NULL, rte_memory_order_relaxed); /* If the queue is empty, the exchange operation is enough to acquire * the lock. Hence, the exchange operation requires acquire semantics. @@ -63,7 +63,7 @@ * visible to other CPUs/threads. Hence, the exchange operation requires * release semantics as well. */ - prev = __atomic_exchange_n(msl, me, __ATOMIC_ACQ_REL); + prev = rte_atomic_exchange_explicit(msl, me, rte_memory_order_acq_rel); if (likely(prev == NULL)) { /* Queue was empty, no further action required, * proceed with lock taken. @@ -77,19 +77,19 @@ * strong as a release fence and is not sufficient to enforce the * desired order here. */ - __atomic_store_n(&prev->next, me, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&prev->next, me, rte_memory_order_release); /* The while-load of me->locked should not move above the previous * store to prev->next. Otherwise it will cause a deadlock. Need a * store-load barrier. */ - __atomic_thread_fence(__ATOMIC_ACQ_REL); + __rte_atomic_thread_fence(rte_memory_order_acq_rel); /* If the lock has already been acquired, it first atomically * places the node at the end of the queue and then proceeds * to spin on me->locked until the previous lock holder resets * the me->locked using mcslock_unlock(). */ - rte_wait_until_equal_32((uint32_t *)&me->locked, 0, __ATOMIC_ACQUIRE); + rte_wait_until_equal_32((uint32_t *)(uintptr_t)&me->locked, 0, rte_memory_order_acquire); } /** @@ -101,34 +101,34 @@ * A pointer to the node of MCS lock passed in rte_mcslock_lock. */ static inline void -rte_mcslock_unlock(rte_mcslock_t **msl, rte_mcslock_t *me) +rte_mcslock_unlock(rte_mcslock_t * __rte_atomic *msl, rte_mcslock_t * __rte_atomic me) { /* Check if there are more nodes in the queue. */ - if (likely(__atomic_load_n(&me->next, __ATOMIC_RELAXED) == NULL)) { + if (likely(rte_atomic_load_explicit(&me->next, rte_memory_order_relaxed) == NULL)) { /* No, last member in the queue. */ - rte_mcslock_t *save_me = __atomic_load_n(&me, __ATOMIC_RELAXED); + rte_mcslock_t *save_me = rte_atomic_load_explicit(&me, rte_memory_order_relaxed); /* Release the lock by setting it to NULL */ - if (likely(__atomic_compare_exchange_n(msl, &save_me, NULL, 0, - __ATOMIC_RELEASE, __ATOMIC_RELAXED))) + if (likely(rte_atomic_compare_exchange_strong_explicit(msl, &save_me, NULL, + rte_memory_order_release, rte_memory_order_relaxed))) return; /* Speculative execution would be allowed to read in the * while-loop first. This has the potential to cause a * deadlock. Need a load barrier. */ - __atomic_thread_fence(__ATOMIC_ACQUIRE); + __rte_atomic_thread_fence(rte_memory_order_acquire); /* More nodes added to the queue by other CPUs. * Wait until the next pointer is set. */ - uintptr_t *next; - next = (uintptr_t *)&me->next; + uintptr_t __rte_atomic *next; + next = (uintptr_t __rte_atomic *)&me->next; RTE_WAIT_UNTIL_MASKED(next, UINTPTR_MAX, !=, 0, - __ATOMIC_RELAXED); + rte_memory_order_relaxed); } /* Pass lock to next waiter. */ - __atomic_store_n(&me->next->locked, 0, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&me->next->locked, 0, rte_memory_order_release); } /** @@ -142,10 +142,10 @@ * 1 if the lock is successfully taken; 0 otherwise. */ static inline int -rte_mcslock_trylock(rte_mcslock_t **msl, rte_mcslock_t *me) +rte_mcslock_trylock(rte_mcslock_t * __rte_atomic *msl, rte_mcslock_t *me) { /* Init me node */ - __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&me->next, NULL, rte_memory_order_relaxed); /* Try to lock */ rte_mcslock_t *expected = NULL; @@ -156,8 +156,8 @@ * is visible to other CPUs/threads. Hence, the compare-exchange * operation requires release semantics as well. */ - return __atomic_compare_exchange_n(msl, &expected, me, 0, - __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + return rte_atomic_compare_exchange_strong_explicit(msl, &expected, me, + rte_memory_order_acq_rel, rte_memory_order_relaxed); } /** @@ -169,9 +169,9 @@ * 1 if the lock is currently taken; 0 otherwise. */ static inline int -rte_mcslock_is_locked(rte_mcslock_t *msl) +rte_mcslock_is_locked(rte_mcslock_t * __rte_atomic msl) { - return (__atomic_load_n(&msl, __ATOMIC_RELAXED) != NULL); + return (rte_atomic_load_explicit(&msl, rte_memory_order_relaxed) != NULL); } #ifdef __cplusplus diff --git a/lib/eal/include/rte_pflock.h b/lib/eal/include/rte_pflock.h index 790be71..a2375b3 100644 --- a/lib/eal/include/rte_pflock.h +++ b/lib/eal/include/rte_pflock.h @@ -41,8 +41,8 @@ */ struct rte_pflock { struct { - uint16_t in; - uint16_t out; + uint16_t __rte_atomic in; + uint16_t __rte_atomic out; } rd, wr; }; typedef struct rte_pflock rte_pflock_t; @@ -117,14 +117,14 @@ struct rte_pflock { * If no writer is present, then the operation has completed * successfully. */ - w = __atomic_fetch_add(&pf->rd.in, RTE_PFLOCK_RINC, __ATOMIC_ACQUIRE) + w = rte_atomic_fetch_add_explicit(&pf->rd.in, RTE_PFLOCK_RINC, rte_memory_order_acquire) & RTE_PFLOCK_WBITS; if (w == 0) return; /* Wait for current write phase to complete. */ RTE_WAIT_UNTIL_MASKED(&pf->rd.in, RTE_PFLOCK_WBITS, !=, w, - __ATOMIC_ACQUIRE); + rte_memory_order_acquire); } /** @@ -140,7 +140,7 @@ struct rte_pflock { static inline void rte_pflock_read_unlock(rte_pflock_t *pf) { - __atomic_fetch_add(&pf->rd.out, RTE_PFLOCK_RINC, __ATOMIC_RELEASE); + rte_atomic_fetch_add_explicit(&pf->rd.out, RTE_PFLOCK_RINC, rte_memory_order_release); } /** @@ -161,8 +161,9 @@ struct rte_pflock { /* Acquire ownership of write-phase. * This is same as rte_ticketlock_lock(). */ - ticket = __atomic_fetch_add(&pf->wr.in, 1, __ATOMIC_RELAXED); - rte_wait_until_equal_16(&pf->wr.out, ticket, __ATOMIC_ACQUIRE); + ticket = rte_atomic_fetch_add_explicit(&pf->wr.in, 1, rte_memory_order_relaxed); + rte_wait_until_equal_16((uint16_t *)(uintptr_t)&pf->wr.out, ticket, + rte_memory_order_acquire); /* * Acquire ticket on read-side in order to allow them @@ -173,10 +174,11 @@ struct rte_pflock { * speculatively. */ w = RTE_PFLOCK_PRES | (ticket & RTE_PFLOCK_PHID); - ticket = __atomic_fetch_add(&pf->rd.in, w, __ATOMIC_RELAXED); + ticket = rte_atomic_fetch_add_explicit(&pf->rd.in, w, rte_memory_order_relaxed); /* Wait for any pending readers to flush. */ - rte_wait_until_equal_16(&pf->rd.out, ticket, __ATOMIC_ACQUIRE); + rte_wait_until_equal_16((uint16_t *)(uintptr_t)&pf->rd.out, ticket, + rte_memory_order_acquire); } /** @@ -193,10 +195,10 @@ struct rte_pflock { rte_pflock_write_unlock(rte_pflock_t *pf) { /* Migrate from write phase to read phase. */ - __atomic_fetch_and(&pf->rd.in, RTE_PFLOCK_LSB, __ATOMIC_RELEASE); + rte_atomic_fetch_and_explicit(&pf->rd.in, RTE_PFLOCK_LSB, rte_memory_order_release); /* Allow other writers to continue. */ - __atomic_fetch_add(&pf->wr.out, 1, __ATOMIC_RELEASE); + rte_atomic_fetch_add_explicit(&pf->wr.out, 1, rte_memory_order_release); } #ifdef __cplusplus diff --git a/lib/eal/include/rte_seqcount.h b/lib/eal/include/rte_seqcount.h index 098af26..a658178 100644 --- a/lib/eal/include/rte_seqcount.h +++ b/lib/eal/include/rte_seqcount.h @@ -32,7 +32,7 @@ * The RTE seqcount type. */ typedef struct { - uint32_t sn; /**< A sequence number for the protected data. */ + uint32_t __rte_atomic sn; /**< A sequence number for the protected data. */ } rte_seqcount_t; /** @@ -106,11 +106,11 @@ static inline uint32_t rte_seqcount_read_begin(const rte_seqcount_t *seqcount) { - /* __ATOMIC_ACQUIRE to prevent loads after (in program order) + /* rte_memory_order_acquire to prevent loads after (in program order) * from happening before the sn load. Synchronizes-with the * store release in rte_seqcount_write_end(). */ - return __atomic_load_n(&seqcount->sn, __ATOMIC_ACQUIRE); + return rte_atomic_load_explicit(&seqcount->sn, rte_memory_order_acquire); } /** @@ -161,9 +161,9 @@ return true; /* make sure the data loads happens before the sn load */ - rte_atomic_thread_fence(__ATOMIC_ACQUIRE); + rte_atomic_thread_fence(rte_memory_order_acquire); - end_sn = __atomic_load_n(&seqcount->sn, __ATOMIC_RELAXED); + end_sn = rte_atomic_load_explicit(&seqcount->sn, rte_memory_order_relaxed); /* A writer incremented the sequence number during this read * critical section. @@ -205,12 +205,12 @@ sn = seqcount->sn + 1; - __atomic_store_n(&seqcount->sn, sn, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&seqcount->sn, sn, rte_memory_order_relaxed); - /* __ATOMIC_RELEASE to prevent stores after (in program order) + /* rte_memory_order_release to prevent stores after (in program order) * from happening before the sn store. */ - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); } /** @@ -237,7 +237,7 @@ sn = seqcount->sn + 1; /* Synchronizes-with the load acquire in rte_seqcount_read_begin(). */ - __atomic_store_n(&seqcount->sn, sn, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&seqcount->sn, sn, rte_memory_order_release); } #ifdef __cplusplus diff --git a/lib/eal/include/rte_ticketlock.h b/lib/eal/include/rte_ticketlock.h index e22d119..d816650 100644 --- a/lib/eal/include/rte_ticketlock.h +++ b/lib/eal/include/rte_ticketlock.h @@ -30,10 +30,10 @@ * The rte_ticketlock_t type. */ typedef union { - uint32_t tickets; + uint32_t __rte_atomic tickets; struct { - uint16_t current; - uint16_t next; + uint16_t __rte_atomic current; + uint16_t __rte_atomic next; } s; } rte_ticketlock_t; @@ -51,7 +51,7 @@ static inline void rte_ticketlock_init(rte_ticketlock_t *tl) { - __atomic_store_n(&tl->tickets, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&tl->tickets, 0, rte_memory_order_relaxed); } /** @@ -63,8 +63,9 @@ static inline void rte_ticketlock_lock(rte_ticketlock_t *tl) { - uint16_t me = __atomic_fetch_add(&tl->s.next, 1, __ATOMIC_RELAXED); - rte_wait_until_equal_16(&tl->s.current, me, __ATOMIC_ACQUIRE); + uint16_t me = rte_atomic_fetch_add_explicit(&tl->s.next, 1, rte_memory_order_relaxed); + rte_wait_until_equal_16((uint16_t *)(uintptr_t)&tl->s.current, me, + rte_memory_order_acquire); } /** @@ -76,8 +77,8 @@ static inline void rte_ticketlock_unlock(rte_ticketlock_t *tl) { - uint16_t i = __atomic_load_n(&tl->s.current, __ATOMIC_RELAXED); - __atomic_store_n(&tl->s.current, i + 1, __ATOMIC_RELEASE); + uint16_t i = rte_atomic_load_explicit(&tl->s.current, rte_memory_order_relaxed); + rte_atomic_store_explicit(&tl->s.current, i + 1, rte_memory_order_release); } /** @@ -92,12 +93,13 @@ rte_ticketlock_trylock(rte_ticketlock_t *tl) { rte_ticketlock_t oldl, newl; - oldl.tickets = __atomic_load_n(&tl->tickets, __ATOMIC_RELAXED); + oldl.tickets = rte_atomic_load_explicit(&tl->tickets, rte_memory_order_relaxed); newl.tickets = oldl.tickets; newl.s.next++; if (oldl.s.next == oldl.s.current) { - if (__atomic_compare_exchange_n(&tl->tickets, &oldl.tickets, - newl.tickets, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) + if (rte_atomic_compare_exchange_strong_explicit(&tl->tickets, + (uint32_t *)(uintptr_t)&oldl.tickets, + newl.tickets, rte_memory_order_acquire, rte_memory_order_relaxed)) return 1; } @@ -116,7 +118,7 @@ rte_ticketlock_is_locked(rte_ticketlock_t *tl) { rte_ticketlock_t tic; - tic.tickets = __atomic_load_n(&tl->tickets, __ATOMIC_ACQUIRE); + tic.tickets = rte_atomic_load_explicit(&tl->tickets, rte_memory_order_acquire); return (tic.s.current != tic.s.next); } @@ -127,7 +129,7 @@ typedef struct { rte_ticketlock_t tl; /**< the actual ticketlock */ - int user; /**< core id using lock, TICKET_LOCK_INVALID_ID for unused */ + int __rte_atomic user; /**< core id using lock, TICKET_LOCK_INVALID_ID for unused */ unsigned int count; /**< count of time this lock has been called */ } rte_ticketlock_recursive_t; @@ -147,7 +149,7 @@ rte_ticketlock_recursive_init(rte_ticketlock_recursive_t *tlr) { rte_ticketlock_init(&tlr->tl); - __atomic_store_n(&tlr->user, TICKET_LOCK_INVALID_ID, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&tlr->user, TICKET_LOCK_INVALID_ID, rte_memory_order_relaxed); tlr->count = 0; } @@ -162,9 +164,9 @@ { int id = rte_gettid(); - if (__atomic_load_n(&tlr->user, __ATOMIC_RELAXED) != id) { + if (rte_atomic_load_explicit(&tlr->user, rte_memory_order_relaxed) != id) { rte_ticketlock_lock(&tlr->tl); - __atomic_store_n(&tlr->user, id, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&tlr->user, id, rte_memory_order_relaxed); } tlr->count++; } @@ -179,8 +181,8 @@ rte_ticketlock_recursive_unlock(rte_ticketlock_recursive_t *tlr) { if (--(tlr->count) == 0) { - __atomic_store_n(&tlr->user, TICKET_LOCK_INVALID_ID, - __ATOMIC_RELAXED); + rte_atomic_store_explicit(&tlr->user, TICKET_LOCK_INVALID_ID, + rte_memory_order_relaxed); rte_ticketlock_unlock(&tlr->tl); } } @@ -198,10 +200,10 @@ { int id = rte_gettid(); - if (__atomic_load_n(&tlr->user, __ATOMIC_RELAXED) != id) { + if (rte_atomic_load_explicit(&tlr->user, rte_memory_order_relaxed) != id) { if (rte_ticketlock_trylock(&tlr->tl) == 0) return 0; - __atomic_store_n(&tlr->user, id, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&tlr->user, id, rte_memory_order_relaxed); } tlr->count++; return 1; diff --git a/lib/eal/include/rte_trace_point.h b/lib/eal/include/rte_trace_point.h index d587591..e682109 100644 --- a/lib/eal/include/rte_trace_point.h +++ b/lib/eal/include/rte_trace_point.h @@ -33,7 +33,7 @@ #include /** The tracepoint object. */ -typedef uint64_t rte_trace_point_t; +typedef uint64_t __rte_atomic rte_trace_point_t; /** * Macro to define the tracepoint arguments in RTE_TRACE_POINT macro. @@ -359,7 +359,7 @@ struct __rte_trace_header { #define __rte_trace_point_emit_header_generic(t) \ void *mem; \ do { \ - const uint64_t val = __atomic_load_n(t, __ATOMIC_ACQUIRE); \ + const uint64_t val = rte_atomic_load_explicit(t, rte_memory_order_acquire); \ if (likely(!(val & __RTE_TRACE_FIELD_ENABLE_MASK))) \ return; \ mem = __rte_trace_mem_get(val); \ diff --git a/lib/eal/loongarch/include/rte_atomic.h b/lib/eal/loongarch/include/rte_atomic.h index 3c82845..0510b8f 100644 --- a/lib/eal/loongarch/include/rte_atomic.h +++ b/lib/eal/loongarch/include/rte_atomic.h @@ -35,9 +35,9 @@ #define rte_io_rmb() rte_mb() static __rte_always_inline void -rte_atomic_thread_fence(int memorder) +rte_atomic_thread_fence(rte_memory_order memorder) { - __atomic_thread_fence(memorder); + __rte_atomic_thread_fence(memorder); } #ifdef __cplusplus diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h index ec8d8a2..7382412 100644 --- a/lib/eal/ppc/include/rte_atomic.h +++ b/lib/eal/ppc/include/rte_atomic.h @@ -38,9 +38,9 @@ #define rte_io_rmb() rte_rmb() static __rte_always_inline void -rte_atomic_thread_fence(int memorder) +rte_atomic_thread_fence(rte_memory_order memorder) { - __atomic_thread_fence(memorder); + __rte_atomic_thread_fence(memorder); } /*------------------------- 16 bit atomic operations -------------------------*/ @@ -48,8 +48,8 @@ static inline int rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src) { - return __atomic_compare_exchange(dst, &exp, &src, 0, __ATOMIC_ACQUIRE, - __ATOMIC_ACQUIRE) ? 1 : 0; + return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire, + rte_memory_order_acquire) ? 1 : 0; } static inline int rte_atomic16_test_and_set(rte_atomic16_t *v) @@ -60,29 +60,29 @@ static inline int rte_atomic16_test_and_set(rte_atomic16_t *v) static inline void rte_atomic16_inc(rte_atomic16_t *v) { - __atomic_fetch_add(&v->cnt, 1, __ATOMIC_ACQUIRE); + rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire); } static inline void rte_atomic16_dec(rte_atomic16_t *v) { - __atomic_fetch_sub(&v->cnt, 1, __ATOMIC_ACQUIRE); + rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire); } static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v) { - return __atomic_fetch_add(&v->cnt, 1, __ATOMIC_ACQUIRE) + 1 == 0; + return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0; } static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) { - return __atomic_fetch_sub(&v->cnt, 1, __ATOMIC_ACQUIRE) - 1 == 0; + return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0; } static inline uint16_t rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val) { - return __atomic_exchange_2(dst, val, __ATOMIC_SEQ_CST); + return __atomic_exchange_2(dst, val, rte_memory_order_seq_cst); } /*------------------------- 32 bit atomic operations -------------------------*/ @@ -90,8 +90,8 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) static inline int rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src) { - return __atomic_compare_exchange(dst, &exp, &src, 0, __ATOMIC_ACQUIRE, - __ATOMIC_ACQUIRE) ? 1 : 0; + return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire, + rte_memory_order_acquire) ? 1 : 0; } static inline int rte_atomic32_test_and_set(rte_atomic32_t *v) @@ -102,29 +102,29 @@ static inline int rte_atomic32_test_and_set(rte_atomic32_t *v) static inline void rte_atomic32_inc(rte_atomic32_t *v) { - __atomic_fetch_add(&v->cnt, 1, __ATOMIC_ACQUIRE); + rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire); } static inline void rte_atomic32_dec(rte_atomic32_t *v) { - __atomic_fetch_sub(&v->cnt, 1, __ATOMIC_ACQUIRE); + rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire); } static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v) { - return __atomic_fetch_add(&v->cnt, 1, __ATOMIC_ACQUIRE) + 1 == 0; + return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0; } static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) { - return __atomic_fetch_sub(&v->cnt, 1, __ATOMIC_ACQUIRE) - 1 == 0; + return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0; } static inline uint32_t rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val) { - return __atomic_exchange_4(dst, val, __ATOMIC_SEQ_CST); + return __atomic_exchange_4(dst, val, rte_memory_order_seq_cst); } /*------------------------- 64 bit atomic operations -------------------------*/ @@ -132,8 +132,8 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) static inline int rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) { - return __atomic_compare_exchange(dst, &exp, &src, 0, __ATOMIC_ACQUIRE, - __ATOMIC_ACQUIRE) ? 1 : 0; + return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire, + rte_memory_order_acquire) ? 1 : 0; } static inline void @@ -157,47 +157,47 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) static inline void rte_atomic64_add(rte_atomic64_t *v, int64_t inc) { - __atomic_fetch_add(&v->cnt, inc, __ATOMIC_ACQUIRE); + rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_acquire); } static inline void rte_atomic64_sub(rte_atomic64_t *v, int64_t dec) { - __atomic_fetch_sub(&v->cnt, dec, __ATOMIC_ACQUIRE); + rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_acquire); } static inline void rte_atomic64_inc(rte_atomic64_t *v) { - __atomic_fetch_add(&v->cnt, 1, __ATOMIC_ACQUIRE); + rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire); } static inline void rte_atomic64_dec(rte_atomic64_t *v) { - __atomic_fetch_sub(&v->cnt, 1, __ATOMIC_ACQUIRE); + rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire); } static inline int64_t rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc) { - return __atomic_fetch_add(&v->cnt, inc, __ATOMIC_ACQUIRE) + inc; + return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_acquire) + inc; } static inline int64_t rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec) { - return __atomic_fetch_sub(&v->cnt, dec, __ATOMIC_ACQUIRE) - dec; + return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_acquire) - dec; } static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v) { - return __atomic_fetch_add(&v->cnt, 1, __ATOMIC_ACQUIRE) + 1 == 0; + return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0; } static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v) { - return __atomic_fetch_sub(&v->cnt, 1, __ATOMIC_ACQUIRE) - 1 == 0; + return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0; } static inline int rte_atomic64_test_and_set(rte_atomic64_t *v) @@ -213,7 +213,7 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v) static inline uint64_t rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val) { - return __atomic_exchange_8(dst, val, __ATOMIC_SEQ_CST); + return __atomic_exchange_8(dst, val, rte_memory_order_seq_cst); } #endif diff --git a/lib/eal/riscv/include/rte_atomic.h b/lib/eal/riscv/include/rte_atomic.h index 4b4633c..2603bc9 100644 --- a/lib/eal/riscv/include/rte_atomic.h +++ b/lib/eal/riscv/include/rte_atomic.h @@ -40,9 +40,9 @@ #define rte_io_rmb() asm volatile("fence ir, ir" : : : "memory") static __rte_always_inline void -rte_atomic_thread_fence(int memorder) +rte_atomic_thread_fence(rte_memory_order memorder) { - __atomic_thread_fence(memorder); + __rte_atomic_thread_fence(memorder); } #ifdef __cplusplus diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h index f2ee1a9..3b3a9a4 100644 --- a/lib/eal/x86/include/rte_atomic.h +++ b/lib/eal/x86/include/rte_atomic.h @@ -82,17 +82,17 @@ /** * Synchronization fence between threads based on the specified memory order. * - * On x86 the __atomic_thread_fence(__ATOMIC_SEQ_CST) generates full 'mfence' + * On x86 the __rte_atomic_thread_fence(rte_memory_order_seq_cst) generates full 'mfence' * which is quite expensive. The optimized implementation of rte_smp_mb is * used instead. */ static __rte_always_inline void -rte_atomic_thread_fence(int memorder) +rte_atomic_thread_fence(rte_memory_order memorder) { - if (memorder == __ATOMIC_SEQ_CST) + if (memorder == rte_memory_order_seq_cst) rte_smp_mb(); else - __atomic_thread_fence(memorder); + __rte_atomic_thread_fence(memorder); } /*------------------------- 16 bit atomic operations -------------------------*/ diff --git a/lib/eal/x86/include/rte_spinlock.h b/lib/eal/x86/include/rte_spinlock.h index 0b20ddf..c76218a 100644 --- a/lib/eal/x86/include/rte_spinlock.h +++ b/lib/eal/x86/include/rte_spinlock.h @@ -78,7 +78,7 @@ static inline int rte_tm_supported(void) } static inline int -rte_try_tm(volatile int *lock) +rte_try_tm(volatile int __rte_atomic *lock) { int i, retries; diff --git a/lib/eal/x86/rte_power_intrinsics.c b/lib/eal/x86/rte_power_intrinsics.c index f749da9..cf70e33 100644 --- a/lib/eal/x86/rte_power_intrinsics.c +++ b/lib/eal/x86/rte_power_intrinsics.c @@ -23,9 +23,9 @@ uint64_t val; /* trigger a write but don't change the value */ - val = __atomic_load_n((volatile uint64_t *)addr, __ATOMIC_RELAXED); - __atomic_compare_exchange_n((volatile uint64_t *)addr, &val, val, 0, - __ATOMIC_RELAXED, __ATOMIC_RELAXED); + val = rte_atomic_load_explicit((volatile uint64_t *)addr, rte_memory_order_relaxed); + rte_atomic_compare_exchange_strong_explicit((volatile uint64_t *)addr, &val, val, + rte_memory_order_relaxed, rte_memory_order_relaxed); } static bool wait_supported;