Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/44352/?format=api
{ "id": 44352, "url": "http://patches.dpdk.org/api/patches/44352/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1536253938-192391-4-git-send-email-honnappa.nagarahalli@arm.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1536253938-192391-4-git-send-email-honnappa.nagarahalli@arm.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1536253938-192391-4-git-send-email-honnappa.nagarahalli@arm.com", "date": "2018-09-06T17:12:17", "name": "[3/4] hash: fix rw concurrency while moving keys", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "aa02c3f70c9b3a0709e5062638ee6740ac9cbbf9", "submitter": { "id": 1045, "url": "http://patches.dpdk.org/api/people/1045/?format=api", "name": "Honnappa Nagarahalli", "email": "honnappa.nagarahalli@arm.com" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1536253938-192391-4-git-send-email-honnappa.nagarahalli@arm.com/mbox/", "series": [ { "id": 1214, "url": "http://patches.dpdk.org/api/series/1214/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=1214", "date": "2018-09-06T17:12:14", "name": "Address reader-writer concurrency in rte_hash", "version": 1, "mbox": "http://patches.dpdk.org/series/1214/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/44352/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/44352/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 0DABF4CE4;\n\tThu, 6 Sep 2018 19:12:36 +0200 (CEST)", "from foss.arm.com (usa-sjc-mx-foss1.foss.arm.com [217.140.101.70])\n\tby dpdk.org (Postfix) with ESMTP id 262084CB3\n\tfor <dev@dpdk.org>; Thu, 6 Sep 2018 19:12:34 +0200 (CEST)", "from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249])\n\tby usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 863127A9;\n\tThu, 6 Sep 2018 10:12:33 -0700 (PDT)", "from 2p2660v4-1.austin.arm.com (2p2660v4-1.austin.arm.com\n\t[10.118.12.164])\n\tby usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id\n\t0FBE33F557; Thu, 6 Sep 2018 10:12:33 -0700 (PDT)" ], "From": "Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>", "To": "bruce.richardson@intel.com,\n\tpablo.de.lara.guarch@intel.com", "Cc": "dev@dpdk.org, honnappa.nagarahalli@dpdk.org, gavin.hu@arm.com,\n\tsteve.capper@arm.com, ola.liljedahl@arm.com, nd@arm.com,\n\tHonnappa Nagarahalli <honnappa.nagarahalli@arm.com>", "Date": "Thu, 6 Sep 2018 12:12:17 -0500", "Message-Id": "<1536253938-192391-4-git-send-email-honnappa.nagarahalli@arm.com>", "X-Mailer": "git-send-email 2.7.4", "In-Reply-To": "<1536253938-192391-1-git-send-email-honnappa.nagarahalli@arm.com>", "References": "<1536253938-192391-1-git-send-email-honnappa.nagarahalli@arm.com>", "Subject": "[dpdk-dev] [PATCH 3/4] hash: fix rw concurrency while moving keys", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Reader-writer concurrency issue, caused by moving the keys\nto their alternative locations during key insert, is solved\nby introducing a global counter(tbl_chng_cnt) indicating a\nchange in table.\n\nSigned-off-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>\nReviewed-by: Gavin Hu <gavin.hu@arm.com>\nReviewed-by: Ola Liljedahl <ola.liljedahl@arm.com>\nReviewed-by: Steve Capper <steve.capper@arm.com>\n---\n lib/librte_hash/rte_cuckoo_hash.c | 307 +++++++++++++++++++++++++-------------\n lib/librte_hash/rte_cuckoo_hash.h | 2 +\n lib/librte_hash/rte_hash.h | 8 +-\n 3 files changed, 206 insertions(+), 111 deletions(-)", "diff": "diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c\nindex 2d89158..1e4a8d4 100644\n--- a/lib/librte_hash/rte_cuckoo_hash.c\n+++ b/lib/librte_hash/rte_cuckoo_hash.c\n@@ -256,6 +256,7 @@ rte_hash_create(const struct rte_hash_parameters *params)\n #endif\n \t/* Setup hash context */\n \tsnprintf(h->name, sizeof(h->name), \"%s\", params->name);\n+\th->tbl_chng_cnt = 0;\n \th->entries = params->entries;\n \th->key_len = params->key_len;\n \th->key_entry_size = key_entry_size;\n@@ -588,7 +589,7 @@ rte_hash_cuckoo_insert_mw(const struct rte_hash *h,\n * return 0 if succeeds.\n */\n static inline int\n-rte_hash_cuckoo_move_insert_mw(const struct rte_hash *h,\n+rte_hash_cuckoo_move_insert_mw(struct rte_hash *h,\n \t\t\tstruct rte_hash_bucket *bkt,\n \t\t\tstruct rte_hash_bucket *alt_bkt,\n \t\t\tconst struct rte_hash_key *key, void *data,\n@@ -639,11 +640,27 @@ rte_hash_cuckoo_move_insert_mw(const struct rte_hash *h,\n \t\tif (unlikely(&h->buckets[prev_alt_bkt_idx]\n \t\t\t\t!= curr_bkt)) {\n \t\t\t/* revert it to empty, otherwise duplicated keys */\n-\t\t\tcurr_bkt->key_idx[curr_slot] = EMPTY_SLOT;\n+\t\t\t__atomic_store_n(&curr_bkt->key_idx[curr_slot],\n+\t\t\t\tEMPTY_SLOT,\n+\t\t\t\t__ATOMIC_RELEASE);\n \t\t\t__hash_rw_writer_unlock(h);\n \t\t\treturn -1;\n \t\t}\n \n+\t\t/* Inform the previous move. The current move need\n+\t\t * not be informed now as the current bucket entry\n+\t\t * is present in both primary and secondary.\n+\t\t * Since there is one writer, load acquires on\n+\t\t * tbl_chng_cnt are not required.\n+\t\t */\n+\t\t__atomic_store_n(&h->tbl_chng_cnt,\n+\t\t\t\t h->tbl_chng_cnt + 1,\n+\t\t\t\t __ATOMIC_RELEASE);\n+\t\t/* The stores to sig_alt and sig_current should not\n+\t\t * move above the store to tbl_chng_cnt.\n+\t\t */\n+\t\t__atomic_thread_fence(__ATOMIC_RELEASE);\n+\n \t\t/* Need to swap current/alt sig to allow later\n \t\t * Cuckoo insert to move elements back to its\n \t\t * primary bucket if available\n@@ -662,6 +679,20 @@ rte_hash_cuckoo_move_insert_mw(const struct rte_hash *h,\n \t\tcurr_bkt = curr_node->bkt;\n \t}\n \n+\t/* Inform the previous move. The current move need\n+\t * not be informed now as the current bucket entry\n+\t * is present in both primary and secondary.\n+\t * Since there is one writer, load acquires on\n+\t * tbl_chng_cnt are not required.\n+\t */\n+\t__atomic_store_n(&h->tbl_chng_cnt,\n+\t\t\t h->tbl_chng_cnt + 1,\n+\t\t\t __ATOMIC_RELEASE);\n+\t/* The stores to sig_alt and sig_current should not\n+\t * move above the store to tbl_chng_cnt.\n+\t */\n+\t__atomic_thread_fence(__ATOMIC_RELEASE);\n+\n \tcurr_bkt->sig_current[curr_slot] = sig;\n \tcurr_bkt->sig_alt[curr_slot] = alt_hash;\n \t/* Release the new bucket entry */\n@@ -680,7 +711,7 @@ rte_hash_cuckoo_move_insert_mw(const struct rte_hash *h,\n * Cuckoo\n */\n static inline int\n-rte_hash_cuckoo_make_space_mw(const struct rte_hash *h,\n+rte_hash_cuckoo_make_space_mw(struct rte_hash *h,\n \t\t\tstruct rte_hash_bucket *bkt,\n \t\t\tstruct rte_hash_bucket *sec_bkt,\n \t\t\tconst struct rte_hash_key *key, void *data,\n@@ -728,7 +759,7 @@ rte_hash_cuckoo_make_space_mw(const struct rte_hash *h,\n }\n \n static inline int32_t\n-__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,\n+__rte_hash_add_key_with_hash(struct rte_hash *h, const void *key,\n \t\t\t\t\t\thash_sig_t sig, void *data)\n {\n \thash_sig_t alt_hash;\n@@ -844,7 +875,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,\n }\n \n int32_t\n-rte_hash_add_key_with_hash(const struct rte_hash *h,\n+rte_hash_add_key_with_hash(struct rte_hash *h,\n \t\t\tconst void *key, hash_sig_t sig)\n {\n \tRETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);\n@@ -852,14 +883,14 @@ rte_hash_add_key_with_hash(const struct rte_hash *h,\n }\n \n int32_t\n-rte_hash_add_key(const struct rte_hash *h, const void *key)\n+rte_hash_add_key(struct rte_hash *h, const void *key)\n {\n \tRETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);\n \treturn __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), 0);\n }\n \n int\n-rte_hash_add_key_with_hash_data(const struct rte_hash *h,\n+rte_hash_add_key_with_hash_data(struct rte_hash *h,\n \t\t\tconst void *key, hash_sig_t sig, void *data)\n {\n \tint ret;\n@@ -873,7 +904,7 @@ rte_hash_add_key_with_hash_data(const struct rte_hash *h,\n }\n \n int\n-rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data)\n+rte_hash_add_key_data(struct rte_hash *h, const void *key, void *data)\n {\n \tint ret;\n \n@@ -926,30 +957,56 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,\n \tuint32_t bucket_idx;\n \thash_sig_t alt_hash;\n \tstruct rte_hash_bucket *bkt;\n+\tuint32_t cnt_b, cnt_a;\n \tint ret;\n \n-\tbucket_idx = sig & h->bucket_bitmask;\n-\tbkt = &h->buckets[bucket_idx];\n-\n \t__hash_rw_reader_lock(h);\n \n-\t/* Check if key is in primary location */\n-\tret = search_one_bucket(h, key, sig, data, bkt);\n-\tif (ret != -1) {\n-\t\t__hash_rw_reader_unlock(h);\n-\t\treturn ret;\n-\t}\n-\t/* Calculate secondary hash */\n-\talt_hash = rte_hash_secondary_hash(sig);\n-\tbucket_idx = alt_hash & h->bucket_bitmask;\n-\tbkt = &h->buckets[bucket_idx];\n+\tdo {\n+\t\t/* Load the table change counter before the lookup\n+\t\t * starts. Acquire semantics will make sure that\n+\t\t * loads in search_one_bucket are not hoisted.\n+\t\t */\n+\t\tcnt_b = __atomic_load_n(&h->tbl_chng_cnt,\n+\t\t\t\t__ATOMIC_ACQUIRE);\n+\n+\t\tbucket_idx = sig & h->bucket_bitmask;\n+\t\tbkt = &h->buckets[bucket_idx];\n+\n+\t\t/* Check if key is in primary location */\n+\t\tret = search_one_bucket(h, key, sig, data, bkt);\n+\t\tif (ret != -1) {\n+\t\t\t__hash_rw_reader_unlock(h);\n+\t\t\treturn ret;\n+\t\t}\n+\t\t/* Calculate secondary hash */\n+\t\talt_hash = rte_hash_secondary_hash(sig);\n+\t\tbucket_idx = alt_hash & h->bucket_bitmask;\n+\t\tbkt = &h->buckets[bucket_idx];\n+\n+\t\t/* Check if key is in secondary location */\n+\t\tret = search_one_bucket(h, key, alt_hash, data, bkt);\n+\t\tif (ret != -1) {\n+\t\t\t__hash_rw_reader_unlock(h);\n+\t\t\treturn ret;\n+\t\t}\n+\n+\t\t/* The loads of sig_current in search_one_bucket\n+\t\t * should not move below the load from tbl_chng_cnt.\n+\t\t */\n+\t\t__atomic_thread_fence(__ATOMIC_ACQUIRE);\n+\t\t/* Re-read the table change counter to check if the\n+\t\t * table has changed during search. If yes, re-do\n+\t\t * the search.\n+\t\t * This load should not get hoisted. The load\n+\t\t * acquires on cnt_b, key index in primary bucket\n+\t\t * and key index in secondary bucket will make sure\n+\t\t * that it does not get hoisted.\n+\t\t */\n+\t\tcnt_a = __atomic_load_n(&h->tbl_chng_cnt,\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t} while (cnt_b != cnt_a);\n \n-\t/* Check if key is in secondary location */\n-\tret = search_one_bucket(h, key, alt_hash, data, bkt);\n-\tif (ret != -1) {\n-\t\t__hash_rw_reader_unlock(h);\n-\t\treturn ret;\n-\t}\n \t__hash_rw_reader_unlock(h);\n \treturn -ENOENT;\n }\n@@ -1190,6 +1247,7 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,\n \tuint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n \tuint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n \tvoid *pdata[RTE_HASH_LOOKUP_BULK_MAX];\n+\tuint32_t cnt_b, cnt_a;\n \n \t/* Prefetch first keys */\n \tfor (i = 0; i < PREFETCH_OFFSET && i < num_keys; i++)\n@@ -1225,102 +1283,137 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,\n \t}\n \n \t__hash_rw_reader_lock(h);\n-\t/* Compare signatures and prefetch key slot of first hit */\n-\tfor (i = 0; i < num_keys; i++) {\n-\t\tcompare_signatures(&prim_hitmask[i], &sec_hitmask[i],\n+\tdo {\n+\t\t/* Load the table change counter before the lookup\n+\t\t * starts. Acquire semantics will make sure that\n+\t\t * loads in compare_signatures are not hoisted.\n+\t\t */\n+\t\tcnt_b = __atomic_load_n(&h->tbl_chng_cnt,\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\n+\t\t/* Compare signatures and prefetch key slot of first hit */\n+\t\tfor (i = 0; i < num_keys; i++) {\n+\t\t\tcompare_signatures(&prim_hitmask[i], &sec_hitmask[i],\n \t\t\t\tprimary_bkt[i], secondary_bkt[i],\n \t\t\t\tprim_hash[i], sec_hash[i], h->sig_cmp_fn);\n \n-\t\tif (prim_hitmask[i]) {\n-\t\t\tuint32_t first_hit = __builtin_ctzl(prim_hitmask[i]);\n-\t\t\tuint32_t key_idx = primary_bkt[i]->key_idx[first_hit];\n-\t\t\tconst struct rte_hash_key *key_slot =\n-\t\t\t\t(const struct rte_hash_key *)(\n-\t\t\t\t(const char *)h->key_store +\n-\t\t\t\tkey_idx * h->key_entry_size);\n-\t\t\trte_prefetch0(key_slot);\n-\t\t\tcontinue;\n-\t\t}\n+\t\t\tif (prim_hitmask[i]) {\n+\t\t\t\tuint32_t first_hit =\n+\t\t\t\t\t\t__builtin_ctzl(prim_hitmask[i]);\n+\t\t\t\tuint32_t key_idx =\n+\t\t\t\t\tprimary_bkt[i]->key_idx[first_hit];\n+\t\t\t\tconst struct rte_hash_key *key_slot =\n+\t\t\t\t\t(const struct rte_hash_key *)(\n+\t\t\t\t\t(const char *)h->key_store +\n+\t\t\t\t\tkey_idx * h->key_entry_size);\n+\t\t\t\trte_prefetch0(key_slot);\n+\t\t\t\tcontinue;\n+\t\t\t}\n \n-\t\tif (sec_hitmask[i]) {\n-\t\t\tuint32_t first_hit = __builtin_ctzl(sec_hitmask[i]);\n-\t\t\tuint32_t key_idx = secondary_bkt[i]->key_idx[first_hit];\n-\t\t\tconst struct rte_hash_key *key_slot =\n-\t\t\t\t(const struct rte_hash_key *)(\n-\t\t\t\t(const char *)h->key_store +\n-\t\t\t\tkey_idx * h->key_entry_size);\n-\t\t\trte_prefetch0(key_slot);\n+\t\t\tif (sec_hitmask[i]) {\n+\t\t\t\tuint32_t first_hit =\n+\t\t\t\t\t\t__builtin_ctzl(sec_hitmask[i]);\n+\t\t\t\tuint32_t key_idx =\n+\t\t\t\t\tsecondary_bkt[i]->key_idx[first_hit];\n+\t\t\t\tconst struct rte_hash_key *key_slot =\n+\t\t\t\t\t(const struct rte_hash_key *)(\n+\t\t\t\t\t(const char *)h->key_store +\n+\t\t\t\t\tkey_idx * h->key_entry_size);\n+\t\t\t\trte_prefetch0(key_slot);\n+\t\t\t}\n \t\t}\n-\t}\n \n-\t/* Compare keys, first hits in primary first */\n-\tfor (i = 0; i < num_keys; i++) {\n-\t\tpositions[i] = -ENOENT;\n-\t\twhile (prim_hitmask[i]) {\n-\t\t\tuint32_t hit_index = __builtin_ctzl(prim_hitmask[i]);\n+\t\t/* Compare keys, first hits in primary first */\n+\t\tfor (i = 0; i < num_keys; i++) {\n+\t\t\tpositions[i] = -ENOENT;\n+\t\t\twhile (prim_hitmask[i]) {\n+\t\t\t\tuint32_t hit_index =\n+\t\t\t\t\t\t__builtin_ctzl(prim_hitmask[i]);\n \n-\t\t\tuint32_t key_idx =\n-\t\t\t__atomic_load_n(\n-\t\t\t\t&primary_bkt[i]->key_idx[hit_index],\n-\t\t\t\t__ATOMIC_ACQUIRE);\n-\t\t\tconst struct rte_hash_key *key_slot =\n-\t\t\t\t(const struct rte_hash_key *)(\n-\t\t\t\t(const char *)h->key_store +\n-\t\t\t\tkey_idx * h->key_entry_size);\n-\n-\t\t\tif (key_idx != EMPTY_SLOT)\n-\t\t\t\tpdata[i] = __atomic_load_n(&key_slot->pdata,\n-\t\t\t\t\t\t__ATOMIC_ACQUIRE);\n-\t\t\t/*\n-\t\t\t * If key index is 0, do not compare key,\n-\t\t\t * as it is checking the dummy slot\n-\t\t\t */\n-\t\t\tif (!!key_idx & !rte_hash_cmp_eq(key_slot->key, keys[i], h)) {\n-\t\t\t\tif (data != NULL)\n-\t\t\t\t\tdata[i] = pdata[i];\n+\t\t\t\tuint32_t key_idx =\n+\t\t\t\t__atomic_load_n(\n+\t\t\t\t\t&primary_bkt[i]->key_idx[hit_index],\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t\t\t\tconst struct rte_hash_key *key_slot =\n+\t\t\t\t\t(const struct rte_hash_key *)(\n+\t\t\t\t\t(const char *)h->key_store +\n+\t\t\t\t\tkey_idx * h->key_entry_size);\n \n-\t\t\t\thits |= 1ULL << i;\n-\t\t\t\tpositions[i] = key_idx - 1;\n-\t\t\t\tgoto next_key;\n+\t\t\t\tif (key_idx != EMPTY_SLOT)\n+\t\t\t\t\tpdata[i] = __atomic_load_n(\n+\t\t\t\t\t\t\t&key_slot->pdata,\n+\t\t\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t\t\t\t/*\n+\t\t\t\t * If key index is 0, do not compare key,\n+\t\t\t\t * as it is checking the dummy slot\n+\t\t\t\t */\n+\t\t\t\tif (!!key_idx &\n+\t\t\t\t\t!rte_hash_cmp_eq(\n+\t\t\t\t\t\tkey_slot->key, keys[i], h)) {\n+\t\t\t\t\tif (data != NULL)\n+\t\t\t\t\t\tdata[i] = pdata[i];\n+\n+\t\t\t\t\thits |= 1ULL << i;\n+\t\t\t\t\tpositions[i] = key_idx - 1;\n+\t\t\t\t\tgoto next_key;\n+\t\t\t\t}\n+\t\t\t\tprim_hitmask[i] &= ~(1 << (hit_index));\n \t\t\t}\n-\t\t\tprim_hitmask[i] &= ~(1 << (hit_index));\n-\t\t}\n \n-\t\twhile (sec_hitmask[i]) {\n-\t\t\tuint32_t hit_index = __builtin_ctzl(sec_hitmask[i]);\n+\t\t\twhile (sec_hitmask[i]) {\n+\t\t\t\tuint32_t hit_index =\n+\t\t\t\t\t\t__builtin_ctzl(sec_hitmask[i]);\n \n-\t\t\tuint32_t key_idx =\n-\t\t\t__atomic_load_n(\n-\t\t\t\t&secondary_bkt[i]->key_idx[hit_index],\n-\t\t\t\t__ATOMIC_ACQUIRE);\n-\t\t\tconst struct rte_hash_key *key_slot =\n-\t\t\t\t(const struct rte_hash_key *)(\n-\t\t\t\t(const char *)h->key_store +\n-\t\t\t\tkey_idx * h->key_entry_size);\n-\n-\t\t\tif (key_idx != EMPTY_SLOT)\n-\t\t\t\tpdata[i] = __atomic_load_n(&key_slot->pdata,\n-\t\t\t\t\t\t__ATOMIC_ACQUIRE);\n-\n-\t\t\t/*\n-\t\t\t * If key index is 0, do not compare key,\n-\t\t\t * as it is checking the dummy slot\n-\t\t\t */\n+\t\t\t\tuint32_t key_idx =\n+\t\t\t\t__atomic_load_n(\n+\t\t\t\t\t&secondary_bkt[i]->key_idx[hit_index],\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t\t\t\tconst struct rte_hash_key *key_slot =\n+\t\t\t\t\t(const struct rte_hash_key *)(\n+\t\t\t\t\t(const char *)h->key_store +\n+\t\t\t\t\tkey_idx * h->key_entry_size);\n \n-\t\t\tif (!!key_idx & !rte_hash_cmp_eq(key_slot->key, keys[i], h)) {\n-\t\t\t\tif (data != NULL)\n-\t\t\t\t\tdata[i] = pdata[i];\n+\t\t\t\tif (key_idx != EMPTY_SLOT)\n+\t\t\t\t\tpdata[i] = __atomic_load_n(\n+\t\t\t\t\t\t\t&key_slot->pdata,\n+\t\t\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t\t\t\t/*\n+\t\t\t\t * If key index is 0, do not compare key,\n+\t\t\t\t * as it is checking the dummy slot\n+\t\t\t\t */\n \n-\t\t\t\thits |= 1ULL << i;\n-\t\t\t\tpositions[i] = key_idx - 1;\n-\t\t\t\tgoto next_key;\n+\t\t\t\tif (!!key_idx &\n+\t\t\t\t\t!rte_hash_cmp_eq(\n+\t\t\t\t\t\tkey_slot->key, keys[i], h)) {\n+\t\t\t\t\tif (data != NULL)\n+\t\t\t\t\t\tdata[i] = pdata[i];\n+\n+\t\t\t\t\thits |= 1ULL << i;\n+\t\t\t\t\tpositions[i] = key_idx - 1;\n+\t\t\t\t\tgoto next_key;\n+\t\t\t\t}\n+\t\t\t\tsec_hitmask[i] &= ~(1 << (hit_index));\n \t\t\t}\n-\t\t\tsec_hitmask[i] &= ~(1 << (hit_index));\n-\t\t}\n \n next_key:\n-\t\tcontinue;\n-\t}\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\t/* The loads of sig_current in compare_signatures\n+\t\t * should not move below the load from tbl_chng_cnt.\n+\t\t */\n+\t\t__atomic_thread_fence(__ATOMIC_ACQUIRE);\n+\t\t/* Re-read the table change counter to check if the\n+\t\t * table has changed during search. If yes, re-do\n+\t\t * the search.\n+\t\t * This load should not get hoisted. The load\n+\t\t * acquires on cnt_b, primary key index and secondary\n+\t\t * key index will make sure that it does not get\n+\t\t * hoisted.\n+\t\t */\n+\t\tcnt_a = __atomic_load_n(&h->tbl_chng_cnt,\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t} while (cnt_b != cnt_a);\n \n \t__hash_rw_reader_unlock(h);\n \ndiff --git a/lib/librte_hash/rte_cuckoo_hash.h b/lib/librte_hash/rte_cuckoo_hash.h\nindex b0c7ef9..5ce864c 100644\n--- a/lib/librte_hash/rte_cuckoo_hash.h\n+++ b/lib/librte_hash/rte_cuckoo_hash.h\n@@ -186,6 +186,8 @@ struct rte_hash {\n \t * to the key table.\n \t */\n \trte_rwlock_t *readwrite_lock; /**< Read-write lock thread-safety. */\n+\tuint32_t tbl_chng_cnt;\n+\t/**< Indicates if the hash table changed from last read. */\n } __rte_cache_aligned;\n \n struct queue_node {\ndiff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h\nindex 9e7d931..05e024b 100644\n--- a/lib/librte_hash/rte_hash.h\n+++ b/lib/librte_hash/rte_hash.h\n@@ -156,7 +156,7 @@ rte_hash_count(const struct rte_hash *h);\n * - -ENOSPC if there is no space in the hash for this key.\n */\n int\n-rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data);\n+rte_hash_add_key_data(struct rte_hash *h, const void *key, void *data);\n \n /**\n * Add a key-value pair with a pre-computed hash value\n@@ -180,7 +180,7 @@ rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data);\n * - -ENOSPC if there is no space in the hash for this key.\n */\n int32_t\n-rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key,\n+rte_hash_add_key_with_hash_data(struct rte_hash *h, const void *key,\n \t\t\t\t\t\thash_sig_t sig, void *data);\n \n /**\n@@ -200,7 +200,7 @@ rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key,\n * array of user data. This value is unique for this key.\n */\n int32_t\n-rte_hash_add_key(const struct rte_hash *h, const void *key);\n+rte_hash_add_key(struct rte_hash *h, const void *key);\n \n /**\n * Add a key to an existing hash table.\n@@ -222,7 +222,7 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);\n * array of user data. This value is unique for this key.\n */\n int32_t\n-rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);\n+rte_hash_add_key_with_hash(struct rte_hash *h, const void *key, hash_sig_t sig);\n \n /**\n * Remove a key from an existing hash table.\n", "prefixes": [ "3/4" ] }