From patchwork Mon Oct 16 23:08:59 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tyler Retzlaff X-Patchwork-Id: 132673 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id CDD3143183; Tue, 17 Oct 2023 01:10:30 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 2071A4113C; Tue, 17 Oct 2023 01:09:23 +0200 (CEST) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mails.dpdk.org (Postfix) with ESMTP id E835740E54 for ; Tue, 17 Oct 2023 01:09:08 +0200 (CEST) Received: by linux.microsoft.com (Postfix, from userid 1086) id A924220B74CF; Mon, 16 Oct 2023 16:09:07 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com A924220B74CF DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1697497747; bh=kQWKjluXmL1Ut6QiK2YkU0t25uV10RyUjIyAgWBMAmI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=E9HaAKhYEuU9lfLoXvNZ2aKDTai6+U9ufmmMHUr/Lp7SA+dxDlV49QrPmnD5jwOKM idqEJEkqGIuybXoEvJx7e8G5Xi50EGyxKx6hE+pGgfq2bbXQmV1ZHTIfczEshQQi+h 2usU7Zviiv+AWT+3A6Jvb0ANFZMikyDz9EDu1ZEk= From: Tyler Retzlaff To: dev@dpdk.org Cc: Akhil Goyal , Anatoly Burakov , Andrew Rybchenko , Bruce Richardson , Chenbo Xia , Ciara Power , David Christensen , David Hunt , Dmitry Kozlyuk , Dmitry Malloy , Elena Agostini , Erik Gabriel Carrillo , Fan Zhang , Ferruh Yigit , Harman Kalra , Harry van Haaren , Honnappa Nagarahalli , Jerin Jacob , Konstantin Ananyev , Matan Azrad , Maxime Coquelin , Narcisa Ana Maria Vasile , Nicolas Chautru , Olivier Matz , Ori Kam , Pallavi Kadam , Pavan Nikhilesh , Reshma Pattan , Sameh Gobriel , Shijith Thotton , Sivaprasad Tummala , Stephen Hemminger , Suanming Mou , Sunil Kumar Kori , Thomas Monjalon , Viacheslav Ovsiienko , Vladimir Medvedkin , Yipeng Wang , Tyler Retzlaff Subject: [PATCH 15/21] vhost: use rte optional stdatomic API Date: Mon, 16 Oct 2023 16:08:59 -0700 Message-Id: <1697497745-20664-16-git-send-email-roretzla@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1697497745-20664-1-git-send-email-roretzla@linux.microsoft.com> References: <1697497745-20664-1-git-send-email-roretzla@linux.microsoft.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API Signed-off-by: Tyler Retzlaff --- lib/vhost/vdpa.c | 3 ++- lib/vhost/vhost.c | 42 ++++++++++++++++---------------- lib/vhost/vhost.h | 39 ++++++++++++++++-------------- lib/vhost/vhost_user.c | 6 ++--- lib/vhost/virtio_net.c | 58 +++++++++++++++++++++++++-------------------- lib/vhost/virtio_net_ctrl.c | 6 +++-- 6 files changed, 84 insertions(+), 70 deletions(-) diff --git a/lib/vhost/vdpa.c b/lib/vhost/vdpa.c index 6284ea2..219eef8 100644 --- a/lib/vhost/vdpa.c +++ b/lib/vhost/vdpa.c @@ -235,7 +235,8 @@ struct rte_vdpa_device * } /* used idx is the synchronization point for the split vring */ - __atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE); + rte_atomic_store_explicit((unsigned short __rte_atomic *)&vq->used->idx, + idx_m, rte_memory_order_release); if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) vring_used_event(s_vring) = idx_m; diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index 7fde412..bdcf85b 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -128,12 +128,13 @@ struct vhost_vq_stats_name_off { { #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) /* - * __sync_ built-ins are deprecated, but __atomic_ ones + * __sync_ built-ins are deprecated, but rte_atomic_ ones * are sub-optimized in older GCC versions. */ __sync_fetch_and_or_1(addr, (1U << nr)); #else - __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); + rte_atomic_fetch_or_explicit((volatile uint8_t __rte_atomic *)addr, (1U << nr), + rte_memory_order_relaxed); #endif } @@ -155,7 +156,7 @@ struct vhost_vq_stats_name_off { return; /* To make sure guest memory updates are committed before logging */ - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); page = addr / VHOST_LOG_PAGE; while (page * VHOST_LOG_PAGE < addr + len) { @@ -197,7 +198,7 @@ struct vhost_vq_stats_name_off { if (unlikely(!vq->log_cache)) return; - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); log_base = (unsigned long *)(uintptr_t)dev->log_base; @@ -206,17 +207,18 @@ struct vhost_vq_stats_name_off { #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) /* - * '__sync' builtins are deprecated, but '__atomic' ones + * '__sync' builtins are deprecated, but 'rte_atomic' ones * are sub-optimized in older GCC versions. */ __sync_fetch_and_or(log_base + elem->offset, elem->val); #else - __atomic_fetch_or(log_base + elem->offset, elem->val, - __ATOMIC_RELAXED); + rte_atomic_fetch_or_explicit( + (unsigned long __rte_atomic *)(log_base + elem->offset), + elem->val, rte_memory_order_relaxed); #endif } - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); vq->log_cache_nb_elem = 0; } @@ -231,7 +233,7 @@ struct vhost_vq_stats_name_off { if (unlikely(!vq->log_cache)) { /* No logging cache allocated, write dirty log map directly */ - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); return; @@ -251,7 +253,7 @@ struct vhost_vq_stats_name_off { * No more room for a new log cache entry, * so write the dirty log map directly. */ - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); return; @@ -1184,11 +1186,11 @@ struct vhost_vq_stats_name_off { if (unlikely(idx >= vq->size)) return -1; - rte_atomic_thread_fence(__ATOMIC_SEQ_CST); + rte_atomic_thread_fence(rte_memory_order_seq_cst); vq->inflight_split->desc[idx].inflight = 0; - rte_atomic_thread_fence(__ATOMIC_SEQ_CST); + rte_atomic_thread_fence(rte_memory_order_seq_cst); vq->inflight_split->used_idx = last_used_idx; return 0; @@ -1227,11 +1229,11 @@ struct vhost_vq_stats_name_off { if (unlikely(head >= vq->size)) return -1; - rte_atomic_thread_fence(__ATOMIC_SEQ_CST); + rte_atomic_thread_fence(rte_memory_order_seq_cst); inflight_info->desc[head].inflight = 0; - rte_atomic_thread_fence(__ATOMIC_SEQ_CST); + rte_atomic_thread_fence(rte_memory_order_seq_cst); inflight_info->old_free_head = inflight_info->free_head; inflight_info->old_used_idx = inflight_info->used_idx; @@ -1454,7 +1456,7 @@ struct vhost_vq_stats_name_off { vq->avail_wrap_counter << 15; } - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); vq->device_event->flags = flags; return 0; @@ -1519,16 +1521,16 @@ struct vhost_vq_stats_name_off { rte_rwlock_read_lock(&vq->access_lock); - __atomic_store_n(&vq->irq_pending, false, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&vq->irq_pending, false, rte_memory_order_release); if (dev->backend_ops->inject_irq(dev, vq)) { if (dev->flags & VIRTIO_DEV_STATS_ENABLED) - __atomic_fetch_add(&vq->stats.guest_notifications_error, - 1, __ATOMIC_RELAXED); + rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications_error, + 1, rte_memory_order_relaxed); } else { if (dev->flags & VIRTIO_DEV_STATS_ENABLED) - __atomic_fetch_add(&vq->stats.guest_notifications, - 1, __ATOMIC_RELAXED); + rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications, + 1, rte_memory_order_relaxed); if (dev->notify_ops->guest_notified) dev->notify_ops->guest_notified(dev->vid); } diff --git a/lib/vhost/vhost.h b/lib/vhost/vhost.h index 5fc9035..f8624fb 100644 --- a/lib/vhost/vhost.h +++ b/lib/vhost/vhost.h @@ -158,9 +158,9 @@ struct virtqueue_stats { uint64_t inflight_completed; uint64_t guest_notifications_suppressed; /* Counters below are atomic, and should be incremented as such. */ - uint64_t guest_notifications; - uint64_t guest_notifications_offloaded; - uint64_t guest_notifications_error; + RTE_ATOMIC(uint64_t) guest_notifications; + RTE_ATOMIC(uint64_t) guest_notifications_offloaded; + RTE_ATOMIC(uint64_t) guest_notifications_error; }; /** @@ -348,7 +348,7 @@ struct vhost_virtqueue { struct vhost_vring_addr ring_addrs; struct virtqueue_stats stats; - bool irq_pending; + RTE_ATOMIC(bool) irq_pending; } __rte_cache_aligned; /* Virtio device status as per Virtio specification */ @@ -486,7 +486,7 @@ struct virtio_net { uint32_t flags; uint16_t vhost_hlen; /* to tell if we need broadcast rarp packet */ - int16_t broadcast_rarp; + RTE_ATOMIC(int16_t) broadcast_rarp; uint32_t nr_vring; int async_copy; @@ -557,7 +557,8 @@ struct virtio_net { static inline bool desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) { - uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE); + uint16_t flags = rte_atomic_load_explicit((unsigned short __rte_atomic *)&desc->flags, + rte_memory_order_acquire); return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) && wrap_counter != !!(flags & VRING_DESC_F_USED); @@ -914,17 +915,19 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, bool expected = false; if (dev->notify_ops->guest_notify) { - if (__atomic_compare_exchange_n(&vq->irq_pending, &expected, true, 0, - __ATOMIC_RELEASE, __ATOMIC_RELAXED)) { + if (rte_atomic_compare_exchange_strong_explicit(&vq->irq_pending, &expected, true, + rte_memory_order_release, rte_memory_order_relaxed)) { if (dev->notify_ops->guest_notify(dev->vid, vq->index)) { if (dev->flags & VIRTIO_DEV_STATS_ENABLED) - __atomic_fetch_add(&vq->stats.guest_notifications_offloaded, - 1, __ATOMIC_RELAXED); + rte_atomic_fetch_add_explicit( + &vq->stats.guest_notifications_offloaded, + 1, rte_memory_order_relaxed); return; } /* Offloading failed, fallback to direct IRQ injection */ - __atomic_store_n(&vq->irq_pending, false, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&vq->irq_pending, false, + rte_memory_order_release); } else { vq->stats.guest_notifications_suppressed++; return; @@ -933,14 +936,14 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, if (dev->backend_ops->inject_irq(dev, vq)) { if (dev->flags & VIRTIO_DEV_STATS_ENABLED) - __atomic_fetch_add(&vq->stats.guest_notifications_error, - 1, __ATOMIC_RELAXED); + rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications_error, + 1, rte_memory_order_relaxed); return; } if (dev->flags & VIRTIO_DEV_STATS_ENABLED) - __atomic_fetch_add(&vq->stats.guest_notifications, - 1, __ATOMIC_RELAXED); + rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications, + 1, rte_memory_order_relaxed); if (dev->notify_ops->guest_notified) dev->notify_ops->guest_notified(dev->vid); } @@ -949,7 +952,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq) { /* Flush used->idx update before we read avail->flags. */ - rte_atomic_thread_fence(__ATOMIC_SEQ_CST); + rte_atomic_thread_fence(rte_memory_order_seq_cst); /* Don't kick guest if we don't reach index specified by guest. */ if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { @@ -981,7 +984,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, bool signalled_used_valid, kick = false; /* Flush used desc update. */ - rte_atomic_thread_fence(__ATOMIC_SEQ_CST); + rte_atomic_thread_fence(rte_memory_order_seq_cst); if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { if (vq->driver_event->flags != @@ -1007,7 +1010,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, goto kick; } - rte_atomic_thread_fence(__ATOMIC_ACQUIRE); + rte_atomic_thread_fence(rte_memory_order_acquire); off_wrap = vq->driver_event->off_wrap; off = off_wrap & ~(1 << 15); diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index 901a80b..e363121 100644 --- a/lib/vhost/vhost_user.c +++ b/lib/vhost/vhost_user.c @@ -1914,7 +1914,7 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev, if (inflight_split->used_idx != used->idx) { inflight_split->desc[last_io].inflight = 0; - rte_atomic_thread_fence(__ATOMIC_SEQ_CST); + rte_atomic_thread_fence(rte_memory_order_seq_cst); inflight_split->used_idx = used->idx; } @@ -2418,10 +2418,10 @@ static int vhost_user_set_log_fd(struct virtio_net **pdev, * Set the flag to inject a RARP broadcast packet at * rte_vhost_dequeue_burst(). * - * __ATOMIC_RELEASE ordering is for making sure the mac is + * rte_memory_order_release ordering is for making sure the mac is * copied before the flag is set. */ - __atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE); + rte_atomic_store_explicit(&dev->broadcast_rarp, 1, rte_memory_order_release); vdpa_dev = dev->vdpa_dev; if (vdpa_dev && vdpa_dev->ops->migration_done) vdpa_dev->ops->migration_done(dev->vid); diff --git a/lib/vhost/virtio_net.c b/lib/vhost/virtio_net.c index 759a78e..8af20f1 100644 --- a/lib/vhost/virtio_net.c +++ b/lib/vhost/virtio_net.c @@ -298,8 +298,8 @@ vhost_log_cache_sync(dev, vq); - __atomic_fetch_add(&vq->used->idx, vq->shadow_used_idx, - __ATOMIC_RELEASE); + rte_atomic_fetch_add_explicit((unsigned short __rte_atomic *)&vq->used->idx, + vq->shadow_used_idx, rte_memory_order_release); vq->shadow_used_idx = 0; vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), sizeof(vq->used->idx)); @@ -335,7 +335,7 @@ } /* The ordering for storing desc flags needs to be enforced. */ - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); for (i = 0; i < vq->shadow_used_idx; i++) { uint16_t flags; @@ -387,8 +387,9 @@ vq->desc_packed[vq->shadow_last_used_idx].id = used_elem->id; /* desc flags is the synchronization point for virtio packed vring */ - __atomic_store_n(&vq->desc_packed[vq->shadow_last_used_idx].flags, - used_elem->flags, __ATOMIC_RELEASE); + rte_atomic_store_explicit( + (unsigned short __rte_atomic *)&vq->desc_packed[vq->shadow_last_used_idx].flags, + used_elem->flags, rte_memory_order_release); vhost_log_cache_used_vring(dev, vq, vq->shadow_last_used_idx * sizeof(struct vring_packed_desc), @@ -418,7 +419,7 @@ desc_base[i].len = lens[i]; } - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { desc_base[i].flags = flags; @@ -515,7 +516,7 @@ vq->desc_packed[vq->last_used_idx + i].len = 0; } - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE) vq->desc_packed[vq->last_used_idx + i].flags = flags; @@ -1415,7 +1416,8 @@ * The ordering between avail index and * desc reads needs to be enforced. */ - avail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE); + avail_head = rte_atomic_load_explicit((unsigned short __rte_atomic *)&vq->avail->idx, + rte_memory_order_acquire); rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); @@ -1806,7 +1808,8 @@ /* * The ordering between avail index and desc reads need to be enforced. */ - avail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE); + avail_head = rte_atomic_load_explicit((unsigned short __rte_atomic *)&vq->avail->idx, + rte_memory_order_acquire); rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); @@ -2222,7 +2225,7 @@ } /* The ordering for storing desc flags needs to be enforced. */ - rte_atomic_thread_fence(__ATOMIC_RELEASE); + rte_atomic_thread_fence(rte_memory_order_release); from = async->last_buffer_idx_packed; @@ -2311,7 +2314,9 @@ vhost_vring_call_packed(dev, vq); } else { write_back_completed_descs_split(vq, n_descs); - __atomic_fetch_add(&vq->used->idx, n_descs, __ATOMIC_RELEASE); + rte_atomic_fetch_add_explicit( + (unsigned short __rte_atomic *)&vq->used->idx, + n_descs, rte_memory_order_release); vhost_vring_call_split(dev, vq); } } else { @@ -3085,8 +3090,8 @@ * The ordering between avail index and * desc reads needs to be enforced. */ - avail_entries = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE) - - vq->last_avail_idx; + avail_entries = rte_atomic_load_explicit((unsigned short __rte_atomic *)&vq->avail->idx, + rte_memory_order_acquire) - vq->last_avail_idx; if (avail_entries == 0) return 0; @@ -3224,7 +3229,7 @@ return -1; } - rte_atomic_thread_fence(__ATOMIC_ACQUIRE); + rte_atomic_thread_fence(rte_memory_order_acquire); vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) lens[i] = descs[avail_idx + i].len; @@ -3297,7 +3302,7 @@ return -1; } - rte_atomic_thread_fence(__ATOMIC_ACQUIRE); + rte_atomic_thread_fence(rte_memory_order_acquire); vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) lens[i] = descs[avail_idx + i].len; @@ -3590,7 +3595,7 @@ * * broadcast_rarp shares a cacheline in the virtio_net structure * with some fields that are accessed during enqueue and - * __atomic_compare_exchange_n causes a write if performed compare + * rte_atomic_compare_exchange_strong_explicit causes a write if performed compare * and exchange. This could result in false sharing between enqueue * and dequeue. * @@ -3598,9 +3603,9 @@ * and only performing compare and exchange if the read indicates it * is likely to be set. */ - if (unlikely(__atomic_load_n(&dev->broadcast_rarp, __ATOMIC_ACQUIRE) && - __atomic_compare_exchange_n(&dev->broadcast_rarp, - &success, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED))) { + if (unlikely(rte_atomic_load_explicit(&dev->broadcast_rarp, rte_memory_order_acquire) && + rte_atomic_compare_exchange_strong_explicit(&dev->broadcast_rarp, + &success, 0, rte_memory_order_release, rte_memory_order_relaxed))) { rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac); if (rarp_mbuf == NULL) { @@ -3683,7 +3688,8 @@ vhost_vring_call_packed(dev, vq); } else { write_back_completed_descs_split(vq, nr_cpl_pkts); - __atomic_fetch_add(&vq->used->idx, nr_cpl_pkts, __ATOMIC_RELEASE); + rte_atomic_fetch_add_explicit((unsigned short __rte_atomic *)&vq->used->idx, + nr_cpl_pkts, rte_memory_order_release); vhost_vring_call_split(dev, vq); } vq->async->pkts_inflight_n -= nr_cpl_pkts; @@ -3714,8 +3720,8 @@ * The ordering between avail index and * desc reads needs to be enforced. */ - avail_entries = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE) - - vq->last_avail_idx; + avail_entries = rte_atomic_load_explicit((unsigned short __rte_atomic *)&vq->avail->idx, + rte_memory_order_acquire) - vq->last_avail_idx; if (avail_entries == 0) goto out; @@ -4204,7 +4210,7 @@ * * broadcast_rarp shares a cacheline in the virtio_net structure * with some fields that are accessed during enqueue and - * __atomic_compare_exchange_n causes a write if performed compare + * rte_atomic_compare_exchange_strong_explicit causes a write if performed compare * and exchange. This could result in false sharing between enqueue * and dequeue. * @@ -4212,9 +4218,9 @@ * and only performing compare and exchange if the read indicates it * is likely to be set. */ - if (unlikely(__atomic_load_n(&dev->broadcast_rarp, __ATOMIC_ACQUIRE) && - __atomic_compare_exchange_n(&dev->broadcast_rarp, - &success, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED))) { + if (unlikely(rte_atomic_load_explicit(&dev->broadcast_rarp, rte_memory_order_acquire) && + rte_atomic_compare_exchange_strong_explicit(&dev->broadcast_rarp, + &success, 0, rte_memory_order_release, rte_memory_order_relaxed))) { rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac); if (rarp_mbuf == NULL) { diff --git a/lib/vhost/virtio_net_ctrl.c b/lib/vhost/virtio_net_ctrl.c index 6b583a0..c4847f8 100644 --- a/lib/vhost/virtio_net_ctrl.c +++ b/lib/vhost/virtio_net_ctrl.c @@ -33,7 +33,8 @@ struct virtio_net_ctrl_elem { uint8_t *ctrl_req; struct vring_desc *descs; - avail_idx = __atomic_load_n(&cvq->avail->idx, __ATOMIC_ACQUIRE); + avail_idx = rte_atomic_load_explicit((unsigned short __rte_atomic *)&cvq->avail->idx, + rte_memory_order_acquire); if (avail_idx == cvq->last_avail_idx) { VHOST_LOG_CONFIG(dev->ifname, DEBUG, "Control queue empty\n"); return 0; @@ -236,7 +237,8 @@ struct virtio_net_ctrl_elem { if (cvq->last_used_idx >= cvq->size) cvq->last_used_idx -= cvq->size; - __atomic_store_n(&cvq->used->idx, cvq->last_used_idx, __ATOMIC_RELEASE); + rte_atomic_store_explicit((unsigned short __rte_atomic *)&cvq->used->idx, + cvq->last_used_idx, rte_memory_order_release); vhost_vring_call_split(dev, dev->cvq);