From patchwork Mon Oct 16 23:09:03 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tyler Retzlaff X-Patchwork-Id: 132679 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1A32A43183; Tue, 17 Oct 2023 01:11:10 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 61F6D427E3; Tue, 17 Oct 2023 01:09:30 +0200 (CEST) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mails.dpdk.org (Postfix) with ESMTP id 33D2440E8A for ; Tue, 17 Oct 2023 01:09:09 +0200 (CEST) Received: by linux.microsoft.com (Postfix, from userid 1086) id EB4CF20B74D3; Mon, 16 Oct 2023 16:09:07 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com EB4CF20B74D3 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1697497747; bh=ARZifBQVDjkWPhexyyha094/2FzY2+ivQcVU9neu3KU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=CM5mgIrHVlOB6GryhnINTAQPn7R/r/imGSJgn3LnqLJjWbNtjgLedOrarXuanxBpZ JTncvER3s8y6GMjrX/wkY79HfhCkwOwekKRbpWkQnFqmwrRbezx2SkJwzwdv5ObZxP wfxsV7lkDnZr/GjEmPF0d94DjUmO2kgCONy444JI= From: Tyler Retzlaff To: dev@dpdk.org Cc: Akhil Goyal , Anatoly Burakov , Andrew Rybchenko , Bruce Richardson , Chenbo Xia , Ciara Power , David Christensen , David Hunt , Dmitry Kozlyuk , Dmitry Malloy , Elena Agostini , Erik Gabriel Carrillo , Fan Zhang , Ferruh Yigit , Harman Kalra , Harry van Haaren , Honnappa Nagarahalli , Jerin Jacob , Konstantin Ananyev , Matan Azrad , Maxime Coquelin , Narcisa Ana Maria Vasile , Nicolas Chautru , Olivier Matz , Ori Kam , Pallavi Kadam , Pavan Nikhilesh , Reshma Pattan , Sameh Gobriel , Shijith Thotton , Sivaprasad Tummala , Stephen Hemminger , Suanming Mou , Sunil Kumar Kori , Thomas Monjalon , Viacheslav Ovsiienko , Vladimir Medvedkin , Yipeng Wang , Tyler Retzlaff Subject: [PATCH 19/21] hash: use rte optional stdatomic API Date: Mon, 16 Oct 2023 16:09:03 -0700 Message-Id: <1697497745-20664-20-git-send-email-roretzla@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1697497745-20664-1-git-send-email-roretzla@linux.microsoft.com> References: <1697497745-20664-1-git-send-email-roretzla@linux.microsoft.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API Signed-off-by: Tyler Retzlaff --- lib/hash/rte_cuckoo_hash.c | 116 ++++++++++++++++++++++----------------------- lib/hash/rte_cuckoo_hash.h | 6 +-- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/lib/hash/rte_cuckoo_hash.c b/lib/hash/rte_cuckoo_hash.c index 19b23f2..b2cf60d 100644 --- a/lib/hash/rte_cuckoo_hash.c +++ b/lib/hash/rte_cuckoo_hash.c @@ -149,7 +149,7 @@ struct rte_hash * unsigned int writer_takes_lock = 0; unsigned int no_free_on_del = 0; uint32_t *ext_bkt_to_free = NULL; - uint32_t *tbl_chng_cnt = NULL; + RTE_ATOMIC(uint32_t) *tbl_chng_cnt = NULL; struct lcore_cache *local_free_slots = NULL; unsigned int readwrite_concur_lf_support = 0; uint32_t i; @@ -713,9 +713,9 @@ struct rte_hash * * variable. Release the application data * to the readers. */ - __atomic_store_n(&k->pdata, + rte_atomic_store_explicit(&k->pdata, data, - __ATOMIC_RELEASE); + rte_memory_order_release); /* * Return index where key is stored, * subtracting the first dummy index @@ -776,9 +776,9 @@ struct rte_hash * * key_idx is the guard variable for signature * and key. */ - __atomic_store_n(&prim_bkt->key_idx[i], + rte_atomic_store_explicit(&prim_bkt->key_idx[i], new_idx, - __ATOMIC_RELEASE); + rte_memory_order_release); break; } } @@ -851,9 +851,9 @@ struct rte_hash * if (unlikely(&h->buckets[prev_alt_bkt_idx] != curr_bkt)) { /* revert it to empty, otherwise duplicated keys */ - __atomic_store_n(&curr_bkt->key_idx[curr_slot], + rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot], EMPTY_SLOT, - __ATOMIC_RELEASE); + rte_memory_order_release); __hash_rw_writer_unlock(h); return -1; } @@ -865,13 +865,13 @@ struct rte_hash * * Since there is one writer, load acquires on * tbl_chng_cnt are not required. */ - __atomic_store_n(h->tbl_chng_cnt, + rte_atomic_store_explicit(h->tbl_chng_cnt, *h->tbl_chng_cnt + 1, - __ATOMIC_RELEASE); + rte_memory_order_release); /* The store to sig_current should not * move above the store to tbl_chng_cnt. */ - __atomic_thread_fence(__ATOMIC_RELEASE); + __atomic_thread_fence(rte_memory_order_release); } /* Need to swap current/alt sig to allow later @@ -881,9 +881,9 @@ struct rte_hash * curr_bkt->sig_current[curr_slot] = prev_bkt->sig_current[prev_slot]; /* Release the updated bucket entry */ - __atomic_store_n(&curr_bkt->key_idx[curr_slot], + rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot], prev_bkt->key_idx[prev_slot], - __ATOMIC_RELEASE); + rte_memory_order_release); curr_slot = prev_slot; curr_node = prev_node; @@ -897,20 +897,20 @@ struct rte_hash * * Since there is one writer, load acquires on * tbl_chng_cnt are not required. */ - __atomic_store_n(h->tbl_chng_cnt, + rte_atomic_store_explicit(h->tbl_chng_cnt, *h->tbl_chng_cnt + 1, - __ATOMIC_RELEASE); + rte_memory_order_release); /* The store to sig_current should not * move above the store to tbl_chng_cnt. */ - __atomic_thread_fence(__ATOMIC_RELEASE); + __atomic_thread_fence(rte_memory_order_release); } curr_bkt->sig_current[curr_slot] = sig; /* Release the new bucket entry */ - __atomic_store_n(&curr_bkt->key_idx[curr_slot], + rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot], new_idx, - __ATOMIC_RELEASE); + rte_memory_order_release); __hash_rw_writer_unlock(h); @@ -1076,9 +1076,9 @@ struct rte_hash * * not leak after the store of pdata in the key store. i.e. pdata is * the guard variable. Release the application data to the readers. */ - __atomic_store_n(&new_k->pdata, + rte_atomic_store_explicit(&new_k->pdata, data, - __ATOMIC_RELEASE); + rte_memory_order_release); /* Copy key */ memcpy(new_k->key, key, h->key_len); @@ -1149,9 +1149,9 @@ struct rte_hash * * key_idx is the guard variable for signature * and key. */ - __atomic_store_n(&cur_bkt->key_idx[i], + rte_atomic_store_explicit(&cur_bkt->key_idx[i], slot_id, - __ATOMIC_RELEASE); + rte_memory_order_release); __hash_rw_writer_unlock(h); return slot_id - 1; } @@ -1185,9 +1185,9 @@ struct rte_hash * * the store to key_idx. i.e. key_idx is the guard variable * for signature and key. */ - __atomic_store_n(&(h->buckets_ext[ext_bkt_id - 1]).key_idx[0], + rte_atomic_store_explicit(&(h->buckets_ext[ext_bkt_id - 1]).key_idx[0], slot_id, - __ATOMIC_RELEASE); + rte_memory_order_release); /* Link the new bucket to sec bucket linked list */ last = rte_hash_get_last_bkt(sec_bkt); last->next = &h->buckets_ext[ext_bkt_id - 1]; @@ -1290,17 +1290,17 @@ struct rte_hash * * key comparison will ensure that the lookup fails. */ if (bkt->sig_current[i] == sig) { - key_idx = __atomic_load_n(&bkt->key_idx[i], - __ATOMIC_ACQUIRE); + key_idx = rte_atomic_load_explicit(&bkt->key_idx[i], + rte_memory_order_acquire); if (key_idx != EMPTY_SLOT) { k = (struct rte_hash_key *) ((char *)keys + key_idx * h->key_entry_size); if (rte_hash_cmp_eq(key, k->key, h) == 0) { if (data != NULL) { - *data = __atomic_load_n( + *data = rte_atomic_load_explicit( &k->pdata, - __ATOMIC_ACQUIRE); + rte_memory_order_acquire); } /* * Return index where key is stored, @@ -1374,8 +1374,8 @@ struct rte_hash * * starts. Acquire semantics will make sure that * loads in search_one_bucket are not hoisted. */ - cnt_b = __atomic_load_n(h->tbl_chng_cnt, - __ATOMIC_ACQUIRE); + cnt_b = rte_atomic_load_explicit(h->tbl_chng_cnt, + rte_memory_order_acquire); /* Check if key is in primary location */ bkt = &h->buckets[prim_bucket_idx]; @@ -1396,7 +1396,7 @@ struct rte_hash * /* The loads of sig_current in search_one_bucket * should not move below the load from tbl_chng_cnt. */ - __atomic_thread_fence(__ATOMIC_ACQUIRE); + __atomic_thread_fence(rte_memory_order_acquire); /* Re-read the table change counter to check if the * table has changed during search. If yes, re-do * the search. @@ -1405,8 +1405,8 @@ struct rte_hash * * and key index in secondary bucket will make sure * that it does not get hoisted. */ - cnt_a = __atomic_load_n(h->tbl_chng_cnt, - __ATOMIC_ACQUIRE); + cnt_a = rte_atomic_load_explicit(h->tbl_chng_cnt, + rte_memory_order_acquire); } while (cnt_b != cnt_a); return -ENOENT; @@ -1611,26 +1611,26 @@ struct rte_hash * for (i = RTE_HASH_BUCKET_ENTRIES - 1; i >= 0; i--) { if (last_bkt->key_idx[i] != EMPTY_SLOT) { cur_bkt->sig_current[pos] = last_bkt->sig_current[i]; - __atomic_store_n(&cur_bkt->key_idx[pos], + rte_atomic_store_explicit(&cur_bkt->key_idx[pos], last_bkt->key_idx[i], - __ATOMIC_RELEASE); + rte_memory_order_release); if (h->readwrite_concur_lf_support) { /* Inform the readers that the table has changed * Since there is one writer, load acquire on * tbl_chng_cnt is not required. */ - __atomic_store_n(h->tbl_chng_cnt, + rte_atomic_store_explicit(h->tbl_chng_cnt, *h->tbl_chng_cnt + 1, - __ATOMIC_RELEASE); + rte_memory_order_release); /* The store to sig_current should * not move above the store to tbl_chng_cnt. */ - __atomic_thread_fence(__ATOMIC_RELEASE); + __atomic_thread_fence(rte_memory_order_release); } last_bkt->sig_current[i] = NULL_SIGNATURE; - __atomic_store_n(&last_bkt->key_idx[i], + rte_atomic_store_explicit(&last_bkt->key_idx[i], EMPTY_SLOT, - __ATOMIC_RELEASE); + rte_memory_order_release); return; } } @@ -1650,8 +1650,8 @@ struct rte_hash * /* Check if key is in bucket */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - key_idx = __atomic_load_n(&bkt->key_idx[i], - __ATOMIC_ACQUIRE); + key_idx = rte_atomic_load_explicit(&bkt->key_idx[i], + rte_memory_order_acquire); if (bkt->sig_current[i] == sig && key_idx != EMPTY_SLOT) { k = (struct rte_hash_key *) ((char *)keys + key_idx * h->key_entry_size); @@ -1663,9 +1663,9 @@ struct rte_hash * if (!h->no_free_on_del) remove_entry(h, bkt, i); - __atomic_store_n(&bkt->key_idx[i], + rte_atomic_store_explicit(&bkt->key_idx[i], EMPTY_SLOT, - __ATOMIC_RELEASE); + rte_memory_order_release); *pos = i; /* @@ -2077,8 +2077,8 @@ struct rte_hash * * starts. Acquire semantics will make sure that * loads in compare_signatures are not hoisted. */ - cnt_b = __atomic_load_n(h->tbl_chng_cnt, - __ATOMIC_ACQUIRE); + cnt_b = rte_atomic_load_explicit(h->tbl_chng_cnt, + rte_memory_order_acquire); /* Compare signatures and prefetch key slot of first hit */ for (i = 0; i < num_keys; i++) { @@ -2121,9 +2121,9 @@ struct rte_hash * __builtin_ctzl(prim_hitmask[i]) >> 1; uint32_t key_idx = - __atomic_load_n( + rte_atomic_load_explicit( &primary_bkt[i]->key_idx[hit_index], - __ATOMIC_ACQUIRE); + rte_memory_order_acquire); const struct rte_hash_key *key_slot = (const struct rte_hash_key *)( (const char *)h->key_store + @@ -2137,9 +2137,9 @@ struct rte_hash * !rte_hash_cmp_eq( key_slot->key, keys[i], h)) { if (data != NULL) - data[i] = __atomic_load_n( + data[i] = rte_atomic_load_explicit( &key_slot->pdata, - __ATOMIC_ACQUIRE); + rte_memory_order_acquire); hits |= 1ULL << i; positions[i] = key_idx - 1; @@ -2153,9 +2153,9 @@ struct rte_hash * __builtin_ctzl(sec_hitmask[i]) >> 1; uint32_t key_idx = - __atomic_load_n( + rte_atomic_load_explicit( &secondary_bkt[i]->key_idx[hit_index], - __ATOMIC_ACQUIRE); + rte_memory_order_acquire); const struct rte_hash_key *key_slot = (const struct rte_hash_key *)( (const char *)h->key_store + @@ -2170,9 +2170,9 @@ struct rte_hash * !rte_hash_cmp_eq( key_slot->key, keys[i], h)) { if (data != NULL) - data[i] = __atomic_load_n( + data[i] = rte_atomic_load_explicit( &key_slot->pdata, - __ATOMIC_ACQUIRE); + rte_memory_order_acquire); hits |= 1ULL << i; positions[i] = key_idx - 1; @@ -2216,7 +2216,7 @@ struct rte_hash * /* The loads of sig_current in compare_signatures * should not move below the load from tbl_chng_cnt. */ - __atomic_thread_fence(__ATOMIC_ACQUIRE); + __atomic_thread_fence(rte_memory_order_acquire); /* Re-read the table change counter to check if the * table has changed during search. If yes, re-do * the search. @@ -2225,8 +2225,8 @@ struct rte_hash * * key index will make sure that it does not get * hoisted. */ - cnt_a = __atomic_load_n(h->tbl_chng_cnt, - __ATOMIC_ACQUIRE); + cnt_a = rte_atomic_load_explicit(h->tbl_chng_cnt, + rte_memory_order_acquire); } while (cnt_b != cnt_a); if (hit_mask != NULL) @@ -2498,8 +2498,8 @@ struct rte_hash * idx = *next % RTE_HASH_BUCKET_ENTRIES; /* If current position is empty, go to the next one */ - while ((position = __atomic_load_n(&h->buckets[bucket_idx].key_idx[idx], - __ATOMIC_ACQUIRE)) == EMPTY_SLOT) { + while ((position = rte_atomic_load_explicit(&h->buckets[bucket_idx].key_idx[idx], + rte_memory_order_acquire)) == EMPTY_SLOT) { (*next)++; /* End of table */ if (*next == total_entries_main) diff --git a/lib/hash/rte_cuckoo_hash.h b/lib/hash/rte_cuckoo_hash.h index eb2644f..f7afc4d 100644 --- a/lib/hash/rte_cuckoo_hash.h +++ b/lib/hash/rte_cuckoo_hash.h @@ -137,7 +137,7 @@ struct lcore_cache { struct rte_hash_key { union { uintptr_t idata; - void *pdata; + RTE_ATOMIC(void *) pdata; }; /* Variable key size */ char key[0]; @@ -155,7 +155,7 @@ enum rte_hash_sig_compare_function { struct rte_hash_bucket { uint16_t sig_current[RTE_HASH_BUCKET_ENTRIES]; - uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES]; + RTE_ATOMIC(uint32_t) key_idx[RTE_HASH_BUCKET_ENTRIES]; uint8_t flag[RTE_HASH_BUCKET_ENTRIES]; @@ -229,7 +229,7 @@ struct rte_hash { * is piggy-backed to freeing of the key index. */ uint32_t *ext_bkt_to_free; - uint32_t *tbl_chng_cnt; + RTE_ATOMIC(uint32_t) *tbl_chng_cnt; /**< Indicates if the hash table changed from last read. */ } __rte_cache_aligned;