From patchwork Mon Oct 16 23:08:49 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tyler Retzlaff X-Patchwork-Id: 132669 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id DB1BF43183; Tue, 17 Oct 2023 01:10:02 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 6E703410EA; Tue, 17 Oct 2023 01:09:18 +0200 (CEST) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mails.dpdk.org (Postfix) with ESMTP id 5211940DCA for ; Tue, 17 Oct 2023 01:09:08 +0200 (CEST) Received: by linux.microsoft.com (Postfix, from userid 1086) id 065C020B74C5; Mon, 16 Oct 2023 16:09:06 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 065C020B74C5 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1697497747; bh=dCxUZBtq3/V86NFcWw7F7aaKcfJwC/v0P68A8nSv4Gc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=auuYxLIR22RamKhqlrh3Hw1umDDhr7tiEJ/8GqdAuz8rFly/j1Ru19HmmKxNBrZYg RUTghGeEUr7DbSzOy/dm4/fWjJpXUWPwhr/iA/tBePqTsoc+VrZSIdxoUHjKHIrYQr xr93wA4PQ+O4I/KacuzoJmylXk241c3YVg9JS5UY= From: Tyler Retzlaff To: dev@dpdk.org Cc: Akhil Goyal , Anatoly Burakov , Andrew Rybchenko , Bruce Richardson , Chenbo Xia , Ciara Power , David Christensen , David Hunt , Dmitry Kozlyuk , Dmitry Malloy , Elena Agostini , Erik Gabriel Carrillo , Fan Zhang , Ferruh Yigit , Harman Kalra , Harry van Haaren , Honnappa Nagarahalli , Jerin Jacob , Konstantin Ananyev , Matan Azrad , Maxime Coquelin , Narcisa Ana Maria Vasile , Nicolas Chautru , Olivier Matz , Ori Kam , Pallavi Kadam , Pavan Nikhilesh , Reshma Pattan , Sameh Gobriel , Shijith Thotton , Sivaprasad Tummala , Stephen Hemminger , Suanming Mou , Sunil Kumar Kori , Thomas Monjalon , Viacheslav Ovsiienko , Vladimir Medvedkin , Yipeng Wang , Tyler Retzlaff Subject: [PATCH 05/21] eal: use rte optional stdatomic API Date: Mon, 16 Oct 2023 16:08:49 -0700 Message-Id: <1697497745-20664-6-git-send-email-roretzla@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1697497745-20664-1-git-send-email-roretzla@linux.microsoft.com> References: <1697497745-20664-1-git-send-email-roretzla@linux.microsoft.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API Signed-off-by: Tyler Retzlaff --- lib/eal/common/eal_common_launch.c | 10 +-- lib/eal/common/eal_common_mcfg.c | 2 +- lib/eal/common/eal_common_proc.c | 14 ++-- lib/eal/common/eal_common_thread.c | 26 ++++---- lib/eal/common/eal_common_trace.c | 8 +-- lib/eal/common/eal_common_trace_ctf.c | 4 +- lib/eal/common/eal_memcfg.h | 2 +- lib/eal/common/eal_private.h | 4 +- lib/eal/common/eal_trace.h | 4 +- lib/eal/common/rte_service.c | 122 +++++++++++++++++----------------- lib/eal/freebsd/eal.c | 20 +++--- lib/eal/include/rte_epoll.h | 3 +- lib/eal/linux/eal.c | 26 ++++---- lib/eal/linux/eal_interrupts.c | 42 ++++++------ lib/eal/ppc/include/rte_atomic.h | 6 +- lib/eal/windows/rte_thread.c | 8 ++- 16 files changed, 152 insertions(+), 149 deletions(-) diff --git a/lib/eal/common/eal_common_launch.c b/lib/eal/common/eal_common_launch.c index 0504598..5320c3b 100644 --- a/lib/eal/common/eal_common_launch.c +++ b/lib/eal/common/eal_common_launch.c @@ -18,8 +18,8 @@ int rte_eal_wait_lcore(unsigned worker_id) { - while (__atomic_load_n(&lcore_config[worker_id].state, - __ATOMIC_ACQUIRE) != WAIT) + while (rte_atomic_load_explicit(&lcore_config[worker_id].state, + rte_memory_order_acquire) != WAIT) rte_pause(); return lcore_config[worker_id].ret; @@ -38,8 +38,8 @@ /* Check if the worker is in 'WAIT' state. Use acquire order * since 'state' variable is used as the guard variable. */ - if (__atomic_load_n(&lcore_config[worker_id].state, - __ATOMIC_ACQUIRE) != WAIT) + if (rte_atomic_load_explicit(&lcore_config[worker_id].state, + rte_memory_order_acquire) != WAIT) goto finish; lcore_config[worker_id].arg = arg; @@ -47,7 +47,7 @@ * before the worker thread starts running the function. * Use worker thread function as the guard variable. */ - __atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&lcore_config[worker_id].f, f, rte_memory_order_release); rc = eal_thread_wake_worker(worker_id); diff --git a/lib/eal/common/eal_common_mcfg.c b/lib/eal/common/eal_common_mcfg.c index 2a785e7..dabb80e 100644 --- a/lib/eal/common/eal_common_mcfg.c +++ b/lib/eal/common/eal_common_mcfg.c @@ -30,7 +30,7 @@ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; /* wait until shared mem_config finish initialising */ - rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, __ATOMIC_RELAXED); + rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, rte_memory_order_relaxed); } int diff --git a/lib/eal/common/eal_common_proc.c b/lib/eal/common/eal_common_proc.c index f20a348..728815c 100644 --- a/lib/eal/common/eal_common_proc.c +++ b/lib/eal/common/eal_common_proc.c @@ -33,7 +33,7 @@ #include "eal_filesystem.h" #include "eal_internal_cfg.h" -static int mp_fd = -1; +static RTE_ATOMIC(int) mp_fd = -1; static rte_thread_t mp_handle_tid; static char mp_filter[PATH_MAX]; /* Filter for secondary process sockets */ static char mp_dir_path[PATH_MAX]; /* The directory path for all mp sockets */ @@ -404,7 +404,7 @@ struct pending_request { struct sockaddr_un sa; int fd; - while ((fd = __atomic_load_n(&mp_fd, __ATOMIC_RELAXED)) >= 0) { + while ((fd = rte_atomic_load_explicit(&mp_fd, rte_memory_order_relaxed)) >= 0) { int ret; ret = read_msg(fd, &msg, &sa); @@ -652,7 +652,7 @@ enum async_action { RTE_LOG(ERR, EAL, "failed to create mp thread: %s\n", strerror(errno)); close(dir_fd); - close(__atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED)); + close(rte_atomic_exchange_explicit(&mp_fd, -1, rte_memory_order_relaxed)); return -1; } @@ -668,7 +668,7 @@ enum async_action { { int fd; - fd = __atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED); + fd = rte_atomic_exchange_explicit(&mp_fd, -1, rte_memory_order_relaxed); if (fd < 0) return; @@ -1282,11 +1282,11 @@ enum mp_status { expected = MP_STATUS_UNKNOWN; desired = status; - if (__atomic_compare_exchange_n(&mcfg->mp_status, &expected, desired, - false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + if (rte_atomic_compare_exchange_strong_explicit(&mcfg->mp_status, &expected, desired, + rte_memory_order_relaxed, rte_memory_order_relaxed)) return true; - return __atomic_load_n(&mcfg->mp_status, __ATOMIC_RELAXED) == desired; + return rte_atomic_load_explicit(&mcfg->mp_status, rte_memory_order_relaxed) == desired; } bool diff --git a/lib/eal/common/eal_common_thread.c b/lib/eal/common/eal_common_thread.c index 668b1ed..c422ea8 100644 --- a/lib/eal/common/eal_common_thread.c +++ b/lib/eal/common/eal_common_thread.c @@ -191,8 +191,8 @@ unsigned rte_socket_id(void) /* Set the state to 'RUNNING'. Use release order * since 'state' variable is used as the guard variable. */ - __atomic_store_n(&lcore_config[lcore_id].state, RUNNING, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&lcore_config[lcore_id].state, RUNNING, + rte_memory_order_release); eal_thread_ack_command(); @@ -201,8 +201,8 @@ unsigned rte_socket_id(void) * are accessed only after update to 'f' is visible. * Wait till the update to 'f' is visible to the worker. */ - while ((f = __atomic_load_n(&lcore_config[lcore_id].f, - __ATOMIC_ACQUIRE)) == NULL) + while ((f = rte_atomic_load_explicit(&lcore_config[lcore_id].f, + rte_memory_order_acquire)) == NULL) rte_pause(); rte_eal_trace_thread_lcore_running(lcore_id, f); @@ -219,8 +219,8 @@ unsigned rte_socket_id(void) * are completed before the state is updated. * Use 'state' as the guard variable. */ - __atomic_store_n(&lcore_config[lcore_id].state, WAIT, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&lcore_config[lcore_id].state, WAIT, + rte_memory_order_release); rte_eal_trace_thread_lcore_stopped(lcore_id); } @@ -242,7 +242,7 @@ struct control_thread_params { /* Control thread status. * If the status is CTRL_THREAD_ERROR, 'ret' has the error code. */ - enum __rte_ctrl_thread_status status; + RTE_ATOMIC(enum __rte_ctrl_thread_status) status; }; static int control_thread_init(void *arg) @@ -259,13 +259,13 @@ static int control_thread_init(void *arg) RTE_PER_LCORE(_socket_id) = SOCKET_ID_ANY; params->ret = rte_thread_set_affinity_by_id(rte_thread_self(), cpuset); if (params->ret != 0) { - __atomic_store_n(¶ms->status, - CTRL_THREAD_ERROR, __ATOMIC_RELEASE); + rte_atomic_store_explicit(¶ms->status, + CTRL_THREAD_ERROR, rte_memory_order_release); return 1; } - __atomic_store_n(¶ms->status, - CTRL_THREAD_RUNNING, __ATOMIC_RELEASE); + rte_atomic_store_explicit(¶ms->status, + CTRL_THREAD_RUNNING, rte_memory_order_release); return 0; } @@ -310,8 +310,8 @@ static uint32_t control_thread_start(void *arg) /* Wait for the control thread to initialize successfully */ while ((ctrl_thread_status = - __atomic_load_n(¶ms->status, - __ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) { + rte_atomic_load_explicit(¶ms->status, + rte_memory_order_acquire)) == CTRL_THREAD_LAUNCHING) { rte_delay_us_sleep(1); } diff --git a/lib/eal/common/eal_common_trace.c b/lib/eal/common/eal_common_trace.c index d2eac2d..6ad87fc 100644 --- a/lib/eal/common/eal_common_trace.c +++ b/lib/eal/common/eal_common_trace.c @@ -97,7 +97,7 @@ struct trace_point_head * bool rte_trace_is_enabled(void) { - return __atomic_load_n(&trace.status, __ATOMIC_ACQUIRE) != 0; + return rte_atomic_load_explicit(&trace.status, rte_memory_order_acquire) != 0; } static void @@ -157,7 +157,7 @@ rte_trace_mode rte_trace_mode_get(void) prev = rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_MASK, rte_memory_order_release); if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) == 0) - __atomic_fetch_add(&trace.status, 1, __ATOMIC_RELEASE); + rte_atomic_fetch_add_explicit(&trace.status, 1, rte_memory_order_release); return 0; } @@ -172,7 +172,7 @@ rte_trace_mode rte_trace_mode_get(void) prev = rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_MASK, rte_memory_order_release); if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) != 0) - __atomic_fetch_sub(&trace.status, 1, __ATOMIC_RELEASE); + rte_atomic_fetch_sub_explicit(&trace.status, 1, rte_memory_order_release); return 0; } @@ -526,7 +526,7 @@ rte_trace_mode rte_trace_mode_get(void) /* Add the trace point at tail */ STAILQ_INSERT_TAIL(&tp_list, tp, next); - __atomic_thread_fence(__ATOMIC_RELEASE); + __atomic_thread_fence(rte_memory_order_release); /* All Good !!! */ return 0; diff --git a/lib/eal/common/eal_common_trace_ctf.c b/lib/eal/common/eal_common_trace_ctf.c index c6775c3..04c4f71 100644 --- a/lib/eal/common/eal_common_trace_ctf.c +++ b/lib/eal/common/eal_common_trace_ctf.c @@ -361,10 +361,10 @@ if (ctf_meta == NULL) return -EINVAL; - if (!__atomic_load_n(&trace->ctf_fixup_done, __ATOMIC_SEQ_CST) && + if (!rte_atomic_load_explicit(&trace->ctf_fixup_done, rte_memory_order_seq_cst) && rte_get_timer_hz()) { meta_fixup(trace, ctf_meta); - __atomic_store_n(&trace->ctf_fixup_done, 1, __ATOMIC_SEQ_CST); + rte_atomic_store_explicit(&trace->ctf_fixup_done, 1, rte_memory_order_seq_cst); } rc = fprintf(f, "%s", ctf_meta); diff --git a/lib/eal/common/eal_memcfg.h b/lib/eal/common/eal_memcfg.h index d5c63e2..60e2089 100644 --- a/lib/eal/common/eal_memcfg.h +++ b/lib/eal/common/eal_memcfg.h @@ -42,7 +42,7 @@ struct rte_mem_config { rte_rwlock_t memory_hotplug_lock; /**< Indicates whether memory hotplug request is in progress. */ - uint8_t mp_status; /**< Multiprocess status. */ + RTE_ATOMIC(uint8_t) mp_status; /**< Multiprocess status. */ /* memory segments and zones */ struct rte_fbarray memzones; /**< Memzone descriptors. */ diff --git a/lib/eal/common/eal_private.h b/lib/eal/common/eal_private.h index ebd496b..4d2e806 100644 --- a/lib/eal/common/eal_private.h +++ b/lib/eal/common/eal_private.h @@ -24,11 +24,11 @@ struct lcore_config { int pipe_main2worker[2]; /**< communication pipe with main */ int pipe_worker2main[2]; /**< communication pipe with main */ - lcore_function_t * volatile f; /**< function to call */ + RTE_ATOMIC(lcore_function_t *) volatile f; /**< function to call */ void * volatile arg; /**< argument of function */ volatile int ret; /**< return value of function */ - volatile enum rte_lcore_state_t state; /**< lcore state */ + volatile RTE_ATOMIC(enum rte_lcore_state_t) state; /**< lcore state */ unsigned int socket_id; /**< physical socket id for this lcore */ unsigned int core_id; /**< core number on socket for this lcore */ int core_index; /**< relative index, starting from 0 */ diff --git a/lib/eal/common/eal_trace.h b/lib/eal/common/eal_trace.h index d66bcfe..ace2ef3 100644 --- a/lib/eal/common/eal_trace.h +++ b/lib/eal/common/eal_trace.h @@ -50,7 +50,7 @@ struct trace_arg { struct trace { char *dir; int register_errno; - uint32_t status; + RTE_ATOMIC(uint32_t) status; enum rte_trace_mode mode; rte_uuid_t uuid; uint32_t buff_len; @@ -65,7 +65,7 @@ struct trace { uint32_t ctf_meta_offset_freq; uint32_t ctf_meta_offset_freq_off_s; uint32_t ctf_meta_offset_freq_off; - uint16_t ctf_fixup_done; + RTE_ATOMIC(uint16_t) ctf_fixup_done; rte_spinlock_t lock; }; diff --git a/lib/eal/common/rte_service.c b/lib/eal/common/rte_service.c index 9e2aa4a..3fc2b9a 100644 --- a/lib/eal/common/rte_service.c +++ b/lib/eal/common/rte_service.c @@ -43,8 +43,8 @@ struct rte_service_spec_impl { rte_spinlock_t execute_lock; /* API set/get-able variables */ - int8_t app_runstate; - int8_t comp_runstate; + RTE_ATOMIC(int8_t) app_runstate; + RTE_ATOMIC(int8_t) comp_runstate; uint8_t internal_flags; /* per service statistics */ @@ -52,24 +52,24 @@ struct rte_service_spec_impl { * It does not indicate the number of cores the service is running * on currently. */ - uint32_t num_mapped_cores; + RTE_ATOMIC(uint32_t) num_mapped_cores; } __rte_cache_aligned; struct service_stats { - uint64_t calls; - uint64_t cycles; + RTE_ATOMIC(uint64_t) calls; + RTE_ATOMIC(uint64_t) cycles; }; /* the internal values of a service core */ struct core_state { /* map of services IDs are run on this core */ uint64_t service_mask; - uint8_t runstate; /* running or stopped */ - uint8_t thread_active; /* indicates when thread is in service_run() */ + RTE_ATOMIC(uint8_t) runstate; /* running or stopped */ + RTE_ATOMIC(uint8_t) thread_active; /* indicates when thread is in service_run() */ uint8_t is_service_core; /* set if core is currently a service core */ uint8_t service_active_on_lcore[RTE_SERVICE_NUM_MAX]; - uint64_t loops; - uint64_t cycles; + RTE_ATOMIC(uint64_t) loops; + RTE_ATOMIC(uint64_t) cycles; struct service_stats service_stats[RTE_SERVICE_NUM_MAX]; } __rte_cache_aligned; @@ -314,11 +314,11 @@ struct core_state { * service_run and service_runstate_get function. */ if (runstate) - __atomic_store_n(&s->comp_runstate, RUNSTATE_RUNNING, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&s->comp_runstate, RUNSTATE_RUNNING, + rte_memory_order_release); else - __atomic_store_n(&s->comp_runstate, RUNSTATE_STOPPED, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&s->comp_runstate, RUNSTATE_STOPPED, + rte_memory_order_release); return 0; } @@ -334,11 +334,11 @@ struct core_state { * service_run runstate_get function. */ if (runstate) - __atomic_store_n(&s->app_runstate, RUNSTATE_RUNNING, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&s->app_runstate, RUNSTATE_RUNNING, + rte_memory_order_release); else - __atomic_store_n(&s->app_runstate, RUNSTATE_STOPPED, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&s->app_runstate, RUNSTATE_STOPPED, + rte_memory_order_release); rte_eal_trace_service_runstate_set(id, runstate); return 0; @@ -354,14 +354,14 @@ struct core_state { * Use load-acquire memory order. This synchronizes with * store-release in service state set functions. */ - if (__atomic_load_n(&s->comp_runstate, __ATOMIC_ACQUIRE) == + if (rte_atomic_load_explicit(&s->comp_runstate, rte_memory_order_acquire) == RUNSTATE_RUNNING && - __atomic_load_n(&s->app_runstate, __ATOMIC_ACQUIRE) == + rte_atomic_load_explicit(&s->app_runstate, rte_memory_order_acquire) == RUNSTATE_RUNNING) { int check_disabled = !(s->internal_flags & SERVICE_F_START_CHECK); - int lcore_mapped = (__atomic_load_n(&s->num_mapped_cores, - __ATOMIC_RELAXED) > 0); + int lcore_mapped = (rte_atomic_load_explicit(&s->num_mapped_cores, + rte_memory_order_relaxed) > 0); return (check_disabled | lcore_mapped); } else @@ -392,15 +392,15 @@ struct core_state { uint64_t end = rte_rdtsc(); uint64_t cycles = end - start; - __atomic_store_n(&cs->cycles, cs->cycles + cycles, - __ATOMIC_RELAXED); - __atomic_store_n(&service_stats->cycles, + rte_atomic_store_explicit(&cs->cycles, cs->cycles + cycles, + rte_memory_order_relaxed); + rte_atomic_store_explicit(&service_stats->cycles, service_stats->cycles + cycles, - __ATOMIC_RELAXED); + rte_memory_order_relaxed); } - __atomic_store_n(&service_stats->calls, - service_stats->calls + 1, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&service_stats->calls, + service_stats->calls + 1, rte_memory_order_relaxed); } else { s->spec.callback(userdata); } @@ -420,9 +420,9 @@ struct core_state { * Use load-acquire memory order. This synchronizes with * store-release in service state set functions. */ - if (__atomic_load_n(&s->comp_runstate, __ATOMIC_ACQUIRE) != + if (rte_atomic_load_explicit(&s->comp_runstate, rte_memory_order_acquire) != RUNSTATE_RUNNING || - __atomic_load_n(&s->app_runstate, __ATOMIC_ACQUIRE) != + rte_atomic_load_explicit(&s->app_runstate, rte_memory_order_acquire) != RUNSTATE_RUNNING || !(service_mask & (UINT64_C(1) << i))) { cs->service_active_on_lcore[i] = 0; @@ -472,11 +472,11 @@ struct core_state { /* Increment num_mapped_cores to reflect that this core is * now mapped capable of running the service. */ - __atomic_fetch_add(&s->num_mapped_cores, 1, __ATOMIC_RELAXED); + rte_atomic_fetch_add_explicit(&s->num_mapped_cores, 1, rte_memory_order_relaxed); int ret = service_run(id, cs, UINT64_MAX, s, serialize_mt_unsafe); - __atomic_fetch_sub(&s->num_mapped_cores, 1, __ATOMIC_RELAXED); + rte_atomic_fetch_sub_explicit(&s->num_mapped_cores, 1, rte_memory_order_relaxed); return ret; } @@ -489,13 +489,13 @@ struct core_state { const int lcore = rte_lcore_id(); struct core_state *cs = &lcore_states[lcore]; - __atomic_store_n(&cs->thread_active, 1, __ATOMIC_SEQ_CST); + rte_atomic_store_explicit(&cs->thread_active, 1, rte_memory_order_seq_cst); /* runstate act as the guard variable. Use load-acquire * memory order here to synchronize with store-release * in runstate update functions. */ - while (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) == + while (rte_atomic_load_explicit(&cs->runstate, rte_memory_order_acquire) == RUNSTATE_RUNNING) { const uint64_t service_mask = cs->service_mask; @@ -513,7 +513,7 @@ struct core_state { service_run(i, cs, service_mask, service_get(i), 1); } - __atomic_store_n(&cs->loops, cs->loops + 1, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&cs->loops, cs->loops + 1, rte_memory_order_relaxed); } /* Switch off this core for all services, to ensure that future @@ -526,7 +526,7 @@ struct core_state { * this store, ensuring that once this store is visible, the service * lcore thread really is done in service cores code. */ - __atomic_store_n(&cs->thread_active, 0, __ATOMIC_SEQ_CST); + rte_atomic_store_explicit(&cs->thread_active, 0, rte_memory_order_seq_cst); return 0; } @@ -539,8 +539,8 @@ struct core_state { /* Load thread_active using ACQUIRE to avoid instructions dependent on * the result being re-ordered before this load completes. */ - return __atomic_load_n(&lcore_states[lcore].thread_active, - __ATOMIC_ACQUIRE); + return rte_atomic_load_explicit(&lcore_states[lcore].thread_active, + rte_memory_order_acquire); } int32_t @@ -646,13 +646,13 @@ struct core_state { if (*set && !lcore_mapped) { lcore_states[lcore].service_mask |= sid_mask; - __atomic_fetch_add(&rte_services[sid].num_mapped_cores, - 1, __ATOMIC_RELAXED); + rte_atomic_fetch_add_explicit(&rte_services[sid].num_mapped_cores, + 1, rte_memory_order_relaxed); } if (!*set && lcore_mapped) { lcore_states[lcore].service_mask &= ~(sid_mask); - __atomic_fetch_sub(&rte_services[sid].num_mapped_cores, - 1, __ATOMIC_RELAXED); + rte_atomic_fetch_sub_explicit(&rte_services[sid].num_mapped_cores, + 1, rte_memory_order_relaxed); } } @@ -709,13 +709,13 @@ struct core_state { * store-release memory order here to synchronize * with load-acquire in runstate read functions. */ - __atomic_store_n(&lcore_states[i].runstate, - RUNSTATE_STOPPED, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&lcore_states[i].runstate, + RUNSTATE_STOPPED, rte_memory_order_release); } } for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) - __atomic_store_n(&rte_services[i].num_mapped_cores, 0, - __ATOMIC_RELAXED); + rte_atomic_store_explicit(&rte_services[i].num_mapped_cores, 0, + rte_memory_order_relaxed); return 0; } @@ -735,8 +735,8 @@ struct core_state { /* Use store-release memory order here to synchronize with * load-acquire in runstate read functions. */ - __atomic_store_n(&lcore_states[lcore].runstate, RUNSTATE_STOPPED, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&lcore_states[lcore].runstate, RUNSTATE_STOPPED, + rte_memory_order_release); return rte_eal_wait_lcore(lcore); } @@ -755,7 +755,7 @@ struct core_state { * memory order here to synchronize with store-release * in runstate update functions. */ - if (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) != + if (rte_atomic_load_explicit(&cs->runstate, rte_memory_order_acquire) != RUNSTATE_STOPPED) return -EBUSY; @@ -779,7 +779,7 @@ struct core_state { * memory order here to synchronize with store-release * in runstate update functions. */ - if (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) == + if (rte_atomic_load_explicit(&cs->runstate, rte_memory_order_acquire) == RUNSTATE_RUNNING) return -EALREADY; @@ -789,7 +789,7 @@ struct core_state { /* Use load-acquire memory order here to synchronize with * store-release in runstate update functions. */ - __atomic_store_n(&cs->runstate, RUNSTATE_RUNNING, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&cs->runstate, RUNSTATE_RUNNING, rte_memory_order_release); rte_eal_trace_service_lcore_start(lcore); @@ -808,7 +808,7 @@ struct core_state { * memory order here to synchronize with store-release * in runstate update functions. */ - if (__atomic_load_n(&lcore_states[lcore].runstate, __ATOMIC_ACQUIRE) == + if (rte_atomic_load_explicit(&lcore_states[lcore].runstate, rte_memory_order_acquire) == RUNSTATE_STOPPED) return -EALREADY; @@ -820,8 +820,8 @@ struct core_state { int32_t enabled = service_mask & (UINT64_C(1) << i); int32_t service_running = rte_service_runstate_get(i); int32_t only_core = (1 == - __atomic_load_n(&rte_services[i].num_mapped_cores, - __ATOMIC_RELAXED)); + rte_atomic_load_explicit(&rte_services[i].num_mapped_cores, + rte_memory_order_relaxed)); /* if the core is mapped, and the service is running, and this * is the only core that is mapped, the service would cease to @@ -834,8 +834,8 @@ struct core_state { /* Use store-release memory order here to synchronize with * load-acquire in runstate read functions. */ - __atomic_store_n(&lcore_states[lcore].runstate, RUNSTATE_STOPPED, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&lcore_states[lcore].runstate, RUNSTATE_STOPPED, + rte_memory_order_release); rte_eal_trace_service_lcore_stop(lcore); @@ -847,7 +847,7 @@ struct core_state { { struct core_state *cs = &lcore_states[lcore]; - return __atomic_load_n(&cs->loops, __ATOMIC_RELAXED); + return rte_atomic_load_explicit(&cs->loops, rte_memory_order_relaxed); } static uint64_t @@ -855,7 +855,7 @@ struct core_state { { struct core_state *cs = &lcore_states[lcore]; - return __atomic_load_n(&cs->cycles, __ATOMIC_RELAXED); + return rte_atomic_load_explicit(&cs->cycles, rte_memory_order_relaxed); } static uint64_t @@ -863,8 +863,8 @@ struct core_state { { struct core_state *cs = &lcore_states[lcore]; - return __atomic_load_n(&cs->service_stats[service_id].calls, - __ATOMIC_RELAXED); + return rte_atomic_load_explicit(&cs->service_stats[service_id].calls, + rte_memory_order_relaxed); } static uint64_t @@ -872,8 +872,8 @@ struct core_state { { struct core_state *cs = &lcore_states[lcore]; - return __atomic_load_n(&cs->service_stats[service_id].cycles, - __ATOMIC_RELAXED); + return rte_atomic_load_explicit(&cs->service_stats[service_id].cycles, + rte_memory_order_relaxed); } typedef uint64_t (*lcore_attr_get_fun)(uint32_t service_id, diff --git a/lib/eal/freebsd/eal.c b/lib/eal/freebsd/eal.c index 39a2868..568e06e 100644 --- a/lib/eal/freebsd/eal.c +++ b/lib/eal/freebsd/eal.c @@ -597,8 +597,8 @@ static void rte_eal_init_alert(const char *msg) return -1; } - if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0, - __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1, + rte_memory_order_relaxed, rte_memory_order_relaxed)) { rte_eal_init_alert("already called initialization."); rte_errno = EALREADY; return -1; @@ -622,7 +622,7 @@ static void rte_eal_init_alert(const char *msg) if (fctret < 0) { rte_eal_init_alert("Invalid 'command line' arguments."); rte_errno = EINVAL; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } @@ -636,20 +636,20 @@ static void rte_eal_init_alert(const char *msg) if (eal_plugins_init() < 0) { rte_eal_init_alert("Cannot init plugins"); rte_errno = EINVAL; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } if (eal_trace_init() < 0) { rte_eal_init_alert("Cannot init trace"); rte_errno = EFAULT; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } if (eal_option_device_parse()) { rte_errno = ENODEV; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } @@ -683,7 +683,7 @@ static void rte_eal_init_alert(const char *msg) if (rte_bus_scan()) { rte_eal_init_alert("Cannot scan the buses for devices"); rte_errno = ENODEV; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } @@ -736,7 +736,7 @@ static void rte_eal_init_alert(const char *msg) if (ret < 0) { rte_eal_init_alert("Cannot get hugepage information."); rte_errno = EACCES; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } } @@ -915,8 +915,8 @@ static void rte_eal_init_alert(const char *msg) static uint32_t run_once; uint32_t has_run = 0; - if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0, - __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1, + rte_memory_order_relaxed, rte_memory_order_relaxed)) { RTE_LOG(WARNING, EAL, "Already called cleanup\n"); rte_errno = EALREADY; return -1; diff --git a/lib/eal/include/rte_epoll.h b/lib/eal/include/rte_epoll.h index 01525f5..ae0cf20 100644 --- a/lib/eal/include/rte_epoll.h +++ b/lib/eal/include/rte_epoll.h @@ -13,6 +13,7 @@ #include +#include #ifdef __cplusplus extern "C" { @@ -38,7 +39,7 @@ enum { /** interrupt epoll event obj, taken by epoll_event.ptr */ struct rte_epoll_event { - uint32_t status; /**< OUT: event status */ + RTE_ATOMIC(uint32_t) status; /**< OUT: event status */ int fd; /**< OUT: event fd */ int epfd; /**< OUT: epoll instance the ev associated with */ struct rte_epoll_data epdata; diff --git a/lib/eal/linux/eal.c b/lib/eal/linux/eal.c index 5f4b2fb..57da058 100644 --- a/lib/eal/linux/eal.c +++ b/lib/eal/linux/eal.c @@ -967,7 +967,7 @@ static void rte_eal_init_alert(const char *msg) rte_eal_init(int argc, char **argv) { int i, fctret, ret; - static uint32_t run_once; + static RTE_ATOMIC(uint32_t) run_once; uint32_t has_run = 0; char cpuset[RTE_CPU_AFFINITY_STR_LEN]; char thread_name[RTE_THREAD_NAME_SIZE]; @@ -983,8 +983,8 @@ static void rte_eal_init_alert(const char *msg) return -1; } - if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0, - __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1, + rte_memory_order_relaxed, rte_memory_order_relaxed)) { rte_eal_init_alert("already called initialization."); rte_errno = EALREADY; return -1; @@ -1008,14 +1008,14 @@ static void rte_eal_init_alert(const char *msg) if (fctret < 0) { rte_eal_init_alert("Invalid 'command line' arguments."); rte_errno = EINVAL; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } if (eal_plugins_init() < 0) { rte_eal_init_alert("Cannot init plugins"); rte_errno = EINVAL; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } @@ -1027,7 +1027,7 @@ static void rte_eal_init_alert(const char *msg) if (eal_option_device_parse()) { rte_errno = ENODEV; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } @@ -1061,7 +1061,7 @@ static void rte_eal_init_alert(const char *msg) if (rte_bus_scan()) { rte_eal_init_alert("Cannot scan the buses for devices"); rte_errno = ENODEV; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } @@ -1125,7 +1125,7 @@ static void rte_eal_init_alert(const char *msg) if (ret < 0) { rte_eal_init_alert("Cannot get hugepage information."); rte_errno = EACCES; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } } @@ -1150,7 +1150,7 @@ static void rte_eal_init_alert(const char *msg) internal_conf->syslog_facility) < 0) { rte_eal_init_alert("Cannot init logging."); rte_errno = ENOMEM; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } @@ -1158,7 +1158,7 @@ static void rte_eal_init_alert(const char *msg) if (rte_eal_vfio_setup() < 0) { rte_eal_init_alert("Cannot init VFIO"); rte_errno = EAGAIN; - __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED); + rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed); return -1; } #endif @@ -1345,11 +1345,11 @@ static void rte_eal_init_alert(const char *msg) int rte_eal_cleanup(void) { - static uint32_t run_once; + static RTE_ATOMIC(uint32_t) run_once; uint32_t has_run = 0; - if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0, - __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1, + rte_memory_order_relaxed, rte_memory_order_relaxed)) { RTE_LOG(WARNING, EAL, "Already called cleanup\n"); rte_errno = EALREADY; return -1; diff --git a/lib/eal/linux/eal_interrupts.c b/lib/eal/linux/eal_interrupts.c index 24fff3d..d4919df 100644 --- a/lib/eal/linux/eal_interrupts.c +++ b/lib/eal/linux/eal_interrupts.c @@ -1266,9 +1266,9 @@ struct rte_intr_source { * ordering below acting as a lock to synchronize * the event data updating. */ - if (!rev || !__atomic_compare_exchange_n(&rev->status, - &valid_status, RTE_EPOLL_EXEC, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) + if (!rev || !rte_atomic_compare_exchange_strong_explicit(&rev->status, + &valid_status, RTE_EPOLL_EXEC, + rte_memory_order_acquire, rte_memory_order_relaxed)) continue; events[count].status = RTE_EPOLL_VALID; @@ -1283,8 +1283,8 @@ struct rte_intr_source { /* the status update should be observed after * the other fields change. */ - __atomic_store_n(&rev->status, RTE_EPOLL_VALID, - __ATOMIC_RELEASE); + rte_atomic_store_explicit(&rev->status, RTE_EPOLL_VALID, + rte_memory_order_release); count++; } return count; @@ -1374,10 +1374,10 @@ struct rte_intr_source { { uint32_t valid_status = RTE_EPOLL_VALID; - while (!__atomic_compare_exchange_n(&ev->status, &valid_status, - RTE_EPOLL_INVALID, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { - while (__atomic_load_n(&ev->status, - __ATOMIC_RELAXED) != RTE_EPOLL_VALID) + while (!rte_atomic_compare_exchange_strong_explicit(&ev->status, &valid_status, + RTE_EPOLL_INVALID, rte_memory_order_acquire, rte_memory_order_relaxed)) { + while (rte_atomic_load_explicit(&ev->status, + rte_memory_order_relaxed) != RTE_EPOLL_VALID) rte_pause(); valid_status = RTE_EPOLL_VALID; } @@ -1402,8 +1402,8 @@ struct rte_intr_source { epfd = rte_intr_tls_epfd(); if (op == EPOLL_CTL_ADD) { - __atomic_store_n(&event->status, RTE_EPOLL_VALID, - __ATOMIC_RELAXED); + rte_atomic_store_explicit(&event->status, RTE_EPOLL_VALID, + rte_memory_order_relaxed); event->fd = fd; /* ignore fd in event */ event->epfd = epfd; ev.data.ptr = (void *)event; @@ -1415,13 +1415,13 @@ struct rte_intr_source { op, fd, strerror(errno)); if (op == EPOLL_CTL_ADD) /* rollback status when CTL_ADD fail */ - __atomic_store_n(&event->status, RTE_EPOLL_INVALID, - __ATOMIC_RELAXED); + rte_atomic_store_explicit(&event->status, RTE_EPOLL_INVALID, + rte_memory_order_relaxed); return -1; } - if (op == EPOLL_CTL_DEL && __atomic_load_n(&event->status, - __ATOMIC_RELAXED) != RTE_EPOLL_INVALID) + if (op == EPOLL_CTL_DEL && rte_atomic_load_explicit(&event->status, + rte_memory_order_relaxed) != RTE_EPOLL_INVALID) eal_epoll_data_safe_free(event); return 0; @@ -1450,8 +1450,8 @@ struct rte_intr_source { case RTE_INTR_EVENT_ADD: epfd_op = EPOLL_CTL_ADD; rev = rte_intr_elist_index_get(intr_handle, efd_idx); - if (__atomic_load_n(&rev->status, - __ATOMIC_RELAXED) != RTE_EPOLL_INVALID) { + if (rte_atomic_load_explicit(&rev->status, + rte_memory_order_relaxed) != RTE_EPOLL_INVALID) { RTE_LOG(INFO, EAL, "Event already been added.\n"); return -EEXIST; } @@ -1474,8 +1474,8 @@ struct rte_intr_source { case RTE_INTR_EVENT_DEL: epfd_op = EPOLL_CTL_DEL; rev = rte_intr_elist_index_get(intr_handle, efd_idx); - if (__atomic_load_n(&rev->status, - __ATOMIC_RELAXED) == RTE_EPOLL_INVALID) { + if (rte_atomic_load_explicit(&rev->status, + rte_memory_order_relaxed) == RTE_EPOLL_INVALID) { RTE_LOG(INFO, EAL, "Event does not exist.\n"); return -EPERM; } @@ -1500,8 +1500,8 @@ struct rte_intr_source { for (i = 0; i < (uint32_t)rte_intr_nb_efd_get(intr_handle); i++) { rev = rte_intr_elist_index_get(intr_handle, i); - if (__atomic_load_n(&rev->status, - __ATOMIC_RELAXED) == RTE_EPOLL_INVALID) + if (rte_atomic_load_explicit(&rev->status, + rte_memory_order_relaxed) == RTE_EPOLL_INVALID) continue; if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) { /* force free if the entry valid */ diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h index 7382412..645c713 100644 --- a/lib/eal/ppc/include/rte_atomic.h +++ b/lib/eal/ppc/include/rte_atomic.h @@ -48,7 +48,7 @@ static inline int rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src) { - return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire, + return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire, rte_memory_order_acquire) ? 1 : 0; } @@ -90,7 +90,7 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) static inline int rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src) { - return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire, + return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire, rte_memory_order_acquire) ? 1 : 0; } @@ -132,7 +132,7 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) static inline int rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) { - return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire, + return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire, rte_memory_order_acquire) ? 1 : 0; } diff --git a/lib/eal/windows/rte_thread.c b/lib/eal/windows/rte_thread.c index acf6484..145ac4b 100644 --- a/lib/eal/windows/rte_thread.c +++ b/lib/eal/windows/rte_thread.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "eal_windows.h" @@ -19,7 +20,7 @@ struct eal_tls_key { struct thread_routine_ctx { rte_thread_func thread_func; - bool thread_init_failed; + RTE_ATOMIC(bool) thread_init_failed; void *routine_args; }; @@ -168,7 +169,8 @@ struct thread_routine_ctx { thread_func_wrapper(void *arg) { struct thread_routine_ctx ctx = *(struct thread_routine_ctx *)arg; - const bool thread_exit = __atomic_load_n(&ctx.thread_init_failed, __ATOMIC_ACQUIRE); + const bool thread_exit = rte_atomic_load_explicit( + &ctx.thread_init_failed, rte_memory_order_acquire); free(arg); @@ -237,7 +239,7 @@ struct thread_routine_ctx { } resume_thread: - __atomic_store_n(&ctx->thread_init_failed, thread_exit, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&ctx->thread_init_failed, thread_exit, rte_memory_order_release); if (ResumeThread(thread_handle) == (DWORD)-1) { ret = thread_log_last_error("ResumeThread()");