Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/137367/?format=api
{ "id": 137367, "url": "http://patches.dpdk.org/api/patches/137367/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20240227174203.2889333-5-yoan.picchi@arm.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20240227174203.2889333-5-yoan.picchi@arm.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20240227174203.2889333-5-yoan.picchi@arm.com", "date": "2024-02-27T17:42:03", "name": "[v5,4/4] hash: add SVE support for bulk key lookup", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "17abacbcf7711b5d5d7f64af143c458e53e7eb0b", "submitter": { "id": 3196, "url": "http://patches.dpdk.org/api/people/3196/?format=api", "name": "Yoan Picchi", "email": "yoan.picchi@arm.com" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20240227174203.2889333-5-yoan.picchi@arm.com/mbox/", "series": [ { "id": 31254, "url": "http://patches.dpdk.org/api/series/31254/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=31254", "date": "2024-02-27T17:42:01", "name": "hash: add SVE support for bulk key lookup", "version": 5, "mbox": "http://patches.dpdk.org/series/31254/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/137367/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/137367/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 4B75543C0D;\n\tTue, 27 Feb 2024 18:42:43 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 3F8A842EF0;\n\tTue, 27 Feb 2024 18:42:23 +0100 (CET)", "from foss.arm.com (foss.arm.com [217.140.110.172])\n by mails.dpdk.org (Postfix) with ESMTP id 306D642E71\n for <dev@dpdk.org>; Tue, 27 Feb 2024 18:42:19 +0100 (CET)", "from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14])\n by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 4E732FEC;\n Tue, 27 Feb 2024 09:42:55 -0800 (PST)", "from octeon10-1.usa.Arm.com (unknown [10.118.91.161])\n by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 24D4E3FB3E;\n Tue, 27 Feb 2024 09:42:16 -0800 (PST)" ], "From": "Yoan Picchi <yoan.picchi@arm.com>", "To": "Yipeng Wang <yipeng1.wang@intel.com>,\n Sameh Gobriel <sameh.gobriel@intel.com>,\n Bruce Richardson <bruce.richardson@intel.com>,\n Vladimir Medvedkin <vladimir.medvedkin@intel.com>", "Cc": "dev@dpdk.org, nd@arm.com, Yoan Picchi <yoan.picchi@arm.com>,\n Harjot Singh <harjot.singh@arm.com>, Nathan Brown <nathan.brown@arm.com>,\n Ruifeng Wang <ruifeng.wang@arm.com>", "Subject": "[PATCH v5 4/4] hash: add SVE support for bulk key lookup", "Date": "Tue, 27 Feb 2024 17:42:03 +0000", "Message-Id": "<20240227174203.2889333-5-yoan.picchi@arm.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20240227174203.2889333-1-yoan.picchi@arm.com>", "References": "<20231020165159.1649282-1-yoan.picchi@arm.com>\n <20240227174203.2889333-1-yoan.picchi@arm.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "- Implemented SVE code for comparing signatures in bulk lookup.\n- Added Defines in code for SVE code support.\n- Optimise NEON code\n- New SVE code is ~5% slower than optimized NEON for N2 processor.\n\nSigned-off-by: Yoan Picchi <yoan.picchi@arm.com>\nSigned-off-by: Harjot Singh <harjot.singh@arm.com>\nReviewed-by: Nathan Brown <nathan.brown@arm.com>\nReviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>\n---\n lib/hash/rte_cuckoo_hash.c | 196 ++++++++++++++++++++++++++++---------\n lib/hash/rte_cuckoo_hash.h | 1 +\n 2 files changed, 151 insertions(+), 46 deletions(-)", "diff": "diff --git a/lib/hash/rte_cuckoo_hash.c b/lib/hash/rte_cuckoo_hash.c\nindex a07dd3a28d..231d6d6ded 100644\n--- a/lib/hash/rte_cuckoo_hash.c\n+++ b/lib/hash/rte_cuckoo_hash.c\n@@ -442,8 +442,11 @@ rte_hash_create(const struct rte_hash_parameters *params)\n \t\th->sig_cmp_fn = RTE_HASH_COMPARE_SSE;\n \telse\n #elif defined(RTE_ARCH_ARM64)\n-\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON))\n+\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {\n \t\th->sig_cmp_fn = RTE_HASH_COMPARE_NEON;\n+\t\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SVE))\n+\t\t\th->sig_cmp_fn = RTE_HASH_COMPARE_SVE;\n+\t}\n \telse\n #endif\n \t\th->sig_cmp_fn = RTE_HASH_COMPARE_SCALAR;\n@@ -1860,37 +1863,103 @@ rte_hash_free_key_with_position(const struct rte_hash *h,\n #if defined(__ARM_NEON)\n \n static inline void\n-compare_signatures_dense(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches,\n-\t\t\tconst struct rte_hash_bucket *prim_bkt,\n-\t\t\tconst struct rte_hash_bucket *sec_bkt,\n+compare_signatures_dense(uint16_t *hitmask_buffer,\n+\t\t\tconst uint16_t *prim_bucket_sigs,\n+\t\t\tconst uint16_t *sec_bucket_sigs,\n \t\t\tuint16_t sig,\n \t\t\tenum rte_hash_sig_compare_function sig_cmp_fn)\n {\n \tunsigned int i;\n \n+\tstatic_assert(sizeof(*hitmask_buffer) >= 2*(RTE_HASH_BUCKET_ENTRIES/8),\n+\t\"The hitmask must be exactly wide enough to accept the whole hitmask if it is dense\");\n+\n \t/* For match mask every bits indicates the match */\n \tswitch (sig_cmp_fn) {\n+#if RTE_HASH_BUCKET_ENTRIES <= 8\n \tcase RTE_HASH_COMPARE_NEON: {\n-\t\tuint16x8_t vmat, x;\n+\t\tuint16x8_t vmat, hit1, hit2;\n \t\tconst uint16x8_t mask = {0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};\n \t\tconst uint16x8_t vsig = vld1q_dup_u16((uint16_t const *)&sig);\n \n \t\t/* Compare all signatures in the primary bucket */\n-\t\tvmat = vceqq_u16(vsig, vld1q_u16((uint16_t const *)prim_bkt->sig_current));\n-\t\tx = vandq_u16(vmat, mask);\n-\t\t*prim_hash_matches = (uint32_t)(vaddvq_u16(x));\n+\t\tvmat = vceqq_u16(vsig, vld1q_u16(prim_bucket_sigs));\n+\t\thit1 = vandq_u16(vmat, mask);\n+\n \t\t/* Compare all signatures in the secondary bucket */\n-\t\tvmat = vceqq_u16(vsig, vld1q_u16((uint16_t const *)sec_bkt->sig_current));\n-\t\tx = vandq_u16(vmat, mask);\n-\t\t*sec_hash_matches = (uint32_t)(vaddvq_u16(x));\n+\t\tvmat = vceqq_u16(vsig, vld1q_u16(sec_bucket_sigs));\n+\t\thit2 = vandq_u16(vmat, mask);\n+\n+\t\thit2 = vshlq_n_u16(hit2, RTE_HASH_BUCKET_ENTRIES);\n+\t\thit2 = vorrq_u16(hit1, hit2);\n+\t\t*hitmask_buffer = vaddvq_u16(hit2);\n+\t\t}\n+\t\tbreak;\n+#endif\n+#if defined(RTE_HAS_SVE_ACLE)\n+\tcase RTE_HASH_COMPARE_SVE: {\n+\t\tsvuint16_t vsign, shift, sv_matches;\n+\t\tsvbool_t pred, match, bucket_wide_pred;\n+\t\tint i = 0;\n+\t\tuint64_t vl = svcnth();\n+\n+\t\tvsign = svdup_u16(sig);\n+\t\tshift = svindex_u16(0, 1);\n+\n+\t\tif (vl >= 2 * RTE_HASH_BUCKET_ENTRIES && RTE_HASH_BUCKET_ENTRIES <= 8) {\n+\t\t\tsvuint16_t primary_array_vect, secondary_array_vect;\n+\t\t\tbucket_wide_pred = svwhilelt_b16(0, RTE_HASH_BUCKET_ENTRIES);\n+\t\t\tprimary_array_vect = svld1_u16(bucket_wide_pred, prim_bucket_sigs);\n+\t\t\tsecondary_array_vect = svld1_u16(bucket_wide_pred, sec_bucket_sigs);\n+\n+\t\t\t/* We merged the two vectors so we can do both comparison at once */\n+\t\t\tprimary_array_vect = svsplice_u16(bucket_wide_pred,\n+\t\t\t\tprimary_array_vect,\n+\t\t\t\tsecondary_array_vect);\n+\t\t\tpred = svwhilelt_b16(0, 2*RTE_HASH_BUCKET_ENTRIES);\n+\n+\t\t\t/* Compare all signatures in the buckets */\n+\t\t\tmatch = svcmpeq_u16(pred, vsign, primary_array_vect);\n+\t\t\tif (svptest_any(svptrue_b16(), match)) {\n+\t\t\t\tsv_matches = svdup_u16(1);\n+\t\t\t\tsv_matches = svlsl_u16_z(match, sv_matches, shift);\n+\t\t\t\t*hitmask_buffer = svorv_u16(svptrue_b16(), sv_matches);\n+\t\t\t}\n+\t\t} else {\n+\t\t\tdo {\n+\t\t\t\tpred = svwhilelt_b16(i, RTE_HASH_BUCKET_ENTRIES);\n+\t\t\t\tuint16_t lower_half = 0;\n+\t\t\t\tuint16_t upper_half = 0;\n+\t\t\t\t/* Compare all signatures in the primary bucket */\n+\t\t\t\tmatch = svcmpeq_u16(pred, vsign, svld1_u16(pred,\n+\t\t\t\t\t\t\t&prim_bucket_sigs[i]));\n+\t\t\t\tif (svptest_any(svptrue_b16(), match)) {\n+\t\t\t\t\tsv_matches = svdup_u16(1);\n+\t\t\t\t\tsv_matches = svlsl_u16_z(match, sv_matches, shift);\n+\t\t\t\t\tlower_half = svorv_u16(svptrue_b16(), sv_matches);\n+\t\t\t\t}\n+\t\t\t\t/* Compare all signatures in the secondary bucket */\n+\t\t\t\tmatch = svcmpeq_u16(pred, vsign, svld1_u16(pred,\n+\t\t\t\t\t\t\t&sec_bucket_sigs[i]));\n+\t\t\t\tif (svptest_any(svptrue_b16(), match)) {\n+\t\t\t\t\tsv_matches = svdup_u16(1);\n+\t\t\t\t\tsv_matches = svlsl_u16_z(match, sv_matches, shift);\n+\t\t\t\t\tupper_half = svorv_u16(svptrue_b16(), sv_matches)\n+\t\t\t\t\t\t<< RTE_HASH_BUCKET_ENTRIES;\n+\t\t\t\t}\n+\t\t\t\thitmask_buffer[i/8] = upper_half | lower_half;\n+\t\t\t\ti += vl;\n+\t\t\t} while (i < RTE_HASH_BUCKET_ENTRIES);\n+\t\t}\n \t\t}\n \t\tbreak;\n+#endif\n \tdefault:\n \t\tfor (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {\n-\t\t\t*prim_hash_matches |=\n-\t\t\t\t((sig == prim_bkt->sig_current[i]) << i);\n-\t\t\t*sec_hash_matches |=\n-\t\t\t\t((sig == sec_bkt->sig_current[i]) << i);\n+\t\t\t*hitmask_buffer |=\n+\t\t\t\t((sig == prim_bucket_sigs[i]) << i);\n+\t\t\t*hitmask_buffer |=\n+\t\t\t\t((sig == sec_bucket_sigs[i]) << i) << RTE_HASH_BUCKET_ENTRIES;\n \t\t}\n \t}\n }\n@@ -1908,7 +1977,7 @@ compare_signatures_sparse(uint32_t *prim_hash_matches, uint32_t *sec_hash_matche\n \n \t/* For match mask the first bit of every two bits indicates the match */\n \tswitch (sig_cmp_fn) {\n-#if defined(__SSE2__)\n+#if defined(__SSE2__) && RTE_HASH_BUCKET_ENTRIES <= 8\n \tcase RTE_HASH_COMPARE_SSE:\n \t\t/* Compare all signatures in the bucket */\n \t\t*prim_hash_matches = _mm_movemask_epi8(_mm_cmpeq_epi16(\n@@ -1948,14 +2017,18 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \tuint64_t hits = 0;\n \tint32_t i;\n \tint32_t ret;\n-\tuint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n-\tuint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n \tstruct rte_hash_bucket *cur_bkt, *next_bkt;\n \n #if defined(__ARM_NEON)\n \tconst int hitmask_padding = 0;\n+\tuint16_t hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n+\n+\tstatic_assert(sizeof(*hitmask_buffer)*8/2 == RTE_HASH_BUCKET_ENTRIES,\n+\t\"The hitmask must be exactly wide enough to accept the whole hitmask when it is dense\");\n #else\n \tconst int hitmask_padding = 1;\n+\tuint32_t prim_hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n+\tuint32_t sec_hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n #endif\n \n \t__hash_rw_reader_lock(h);\n@@ -1963,18 +2036,24 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t/* Compare signatures and prefetch key slot of first hit */\n \tfor (i = 0; i < num_keys; i++) {\n #if defined(__ARM_NEON)\n-\t\tcompare_signatures_dense(&prim_hitmask[i], &sec_hitmask[i],\n-\t\t\tprimary_bkt[i], secondary_bkt[i],\n+\t\tuint16_t *hitmask = &hitmask_buffer[i];\n+\t\tcompare_signatures_dense(hitmask,\n+\t\t\tprimary_bkt[i]->sig_current,\n+\t\t\tsecondary_bkt[i]->sig_current,\n \t\t\tsig[i], h->sig_cmp_fn);\n+\t\tconst unsigned int prim_hitmask = *(uint8_t *)(hitmask);\n+\t\tconst unsigned int sec_hitmask = *((uint8_t *)(hitmask)+1);\n #else\n-\t\tcompare_signatures_sparse(&prim_hitmask[i], &sec_hitmask[i],\n+\t\tcompare_signatures_sparse(&prim_hitmask_buffer[i], &sec_hitmask_buffer[i],\n \t\t\tprimary_bkt[i], secondary_bkt[i],\n \t\t\tsig[i], h->sig_cmp_fn);\n+\t\tconst unsigned int prim_hitmask = prim_hitmask_buffer[i];\n+\t\tconst unsigned int sec_hitmask = sec_hitmask_buffer[i];\n #endif\n \n-\t\tif (prim_hitmask[i]) {\n+\t\tif (prim_hitmask) {\n \t\t\tuint32_t first_hit =\n-\t\t\t\t\trte_ctz32(prim_hitmask[i])\n+\t\t\t\t\trte_ctz32(prim_hitmask)\n \t\t\t\t\t>> hitmask_padding;\n \t\t\tuint32_t key_idx =\n \t\t\t\tprimary_bkt[i]->key_idx[first_hit];\n@@ -1986,9 +2065,9 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t\t\tcontinue;\n \t\t}\n \n-\t\tif (sec_hitmask[i]) {\n+\t\tif (sec_hitmask) {\n \t\t\tuint32_t first_hit =\n-\t\t\t\t\trte_ctz32(sec_hitmask[i])\n+\t\t\t\t\trte_ctz32(sec_hitmask)\n \t\t\t\t\t>> hitmask_padding;\n \t\t\tuint32_t key_idx =\n \t\t\t\tsecondary_bkt[i]->key_idx[first_hit];\n@@ -2003,9 +2082,17 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t/* Compare keys, first hits in primary first */\n \tfor (i = 0; i < num_keys; i++) {\n \t\tpositions[i] = -ENOENT;\n-\t\twhile (prim_hitmask[i]) {\n+#if defined(__ARM_NEON)\n+\t\tuint16_t *hitmask = &hitmask_buffer[i];\n+\t\tunsigned int prim_hitmask = *(uint8_t *)(hitmask);\n+\t\tunsigned int sec_hitmask = *((uint8_t *)(hitmask)+1);\n+#else\n+\t\tunsigned int prim_hitmask = prim_hitmask_buffer[i];\n+\t\tunsigned int sec_hitmask = sec_hitmask_buffer[i];\n+#endif\n+\t\twhile (prim_hitmask) {\n \t\t\tuint32_t hit_index =\n-\t\t\t\t\trte_ctz32(prim_hitmask[i])\n+\t\t\t\t\trte_ctz32(prim_hitmask)\n \t\t\t\t\t>> hitmask_padding;\n \t\t\tuint32_t key_idx =\n \t\t\t\tprimary_bkt[i]->key_idx[hit_index];\n@@ -2028,12 +2115,12 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t\t\t\tpositions[i] = key_idx - 1;\n \t\t\t\tgoto next_key;\n \t\t\t}\n-\t\t\tprim_hitmask[i] &= ~(1 << (hit_index << hitmask_padding));\n+\t\t\tprim_hitmask &= ~(1 << (hit_index << hitmask_padding));\n \t\t}\n \n-\t\twhile (sec_hitmask[i]) {\n+\t\twhile (sec_hitmask) {\n \t\t\tuint32_t hit_index =\n-\t\t\t\t\trte_ctz32(sec_hitmask[i])\n+\t\t\t\t\trte_ctz32(sec_hitmask)\n \t\t\t\t\t>> hitmask_padding;\n \t\t\tuint32_t key_idx =\n \t\t\t\tsecondary_bkt[i]->key_idx[hit_index];\n@@ -2057,7 +2144,7 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t\t\t\tpositions[i] = key_idx - 1;\n \t\t\t\tgoto next_key;\n \t\t\t}\n-\t\t\tsec_hitmask[i] &= ~(1 << (hit_index << hitmask_padding));\n+\t\t\tsec_hitmask &= ~(1 << (hit_index << hitmask_padding));\n \t\t}\n next_key:\n \t\tcontinue;\n@@ -2107,15 +2194,18 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \tuint64_t hits = 0;\n \tint32_t i;\n \tint32_t ret;\n-\tuint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n-\tuint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n \tstruct rte_hash_bucket *cur_bkt, *next_bkt;\n \tuint32_t cnt_b, cnt_a;\n \n #if defined(__ARM_NEON)\n \tconst int hitmask_padding = 0;\n+\tuint16_t hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n+\tstatic_assert(sizeof(*hitmask_buffer)*8/2 == RTE_HASH_BUCKET_ENTRIES,\n+\t\"The hitmask must be exactly wide enough to accept the whole hitmask chen it is dense\");\n #else\n \tconst int hitmask_padding = 1;\n+\tuint32_t prim_hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n+\tuint32_t sec_hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n #endif\n \n \tfor (i = 0; i < num_keys; i++)\n@@ -2132,18 +2222,24 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \t\t/* Compare signatures and prefetch key slot of first hit */\n \t\tfor (i = 0; i < num_keys; i++) {\n #if defined(__ARM_NEON)\n-\t\t\tcompare_signatures_dense(&prim_hitmask[i], &sec_hitmask[i],\n-\t\t\t\tprimary_bkt[i], secondary_bkt[i],\n+\t\t\tuint16_t *hitmask = &hitmask_buffer[i];\n+\t\t\tcompare_signatures_dense(hitmask,\n+\t\t\t\tprimary_bkt[i]->sig_current,\n+\t\t\t\tsecondary_bkt[i]->sig_current,\n \t\t\t\tsig[i], h->sig_cmp_fn);\n+\t\t\tconst unsigned int prim_hitmask = *(uint8_t *)(hitmask);\n+\t\t\tconst unsigned int sec_hitmask = *((uint8_t *)(hitmask)+1);\n #else\n-\t\t\tcompare_signatures_sparse(&prim_hitmask[i], &sec_hitmask[i],\n+\t\t\tcompare_signatures_sparse(&prim_hitmask_buffer[i], &sec_hitmask_buffer[i],\n \t\t\t\tprimary_bkt[i], secondary_bkt[i],\n \t\t\t\tsig[i], h->sig_cmp_fn);\n+\t\t\tconst unsigned int prim_hitmask = prim_hitmask_buffer[i];\n+\t\t\tconst unsigned int sec_hitmask = sec_hitmask_buffer[i];\n #endif\n \n-\t\t\tif (prim_hitmask[i]) {\n+\t\t\tif (prim_hitmask) {\n \t\t\t\tuint32_t first_hit =\n-\t\t\t\t\t\trte_ctz32(prim_hitmask[i])\n+\t\t\t\t\t\trte_ctz32(prim_hitmask)\n \t\t\t\t\t\t>> hitmask_padding;\n \t\t\t\tuint32_t key_idx =\n \t\t\t\t\tprimary_bkt[i]->key_idx[first_hit];\n@@ -2155,9 +2251,9 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \t\t\t\tcontinue;\n \t\t\t}\n \n-\t\t\tif (sec_hitmask[i]) {\n+\t\t\tif (sec_hitmask) {\n \t\t\t\tuint32_t first_hit =\n-\t\t\t\t\t\trte_ctz32(sec_hitmask[i])\n+\t\t\t\t\t\trte_ctz32(sec_hitmask)\n \t\t\t\t\t\t>> hitmask_padding;\n \t\t\t\tuint32_t key_idx =\n \t\t\t\t\tsecondary_bkt[i]->key_idx[first_hit];\n@@ -2171,9 +2267,17 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \n \t\t/* Compare keys, first hits in primary first */\n \t\tfor (i = 0; i < num_keys; i++) {\n-\t\t\twhile (prim_hitmask[i]) {\n+#if defined(__ARM_NEON)\n+\t\t\tuint16_t *hitmask = &hitmask_buffer[i];\n+\t\t\tunsigned int prim_hitmask = *(uint8_t *)(hitmask);\n+\t\t\tunsigned int sec_hitmask = *((uint8_t *)(hitmask)+1);\n+#else\n+\t\t\tunsigned int prim_hitmask = prim_hitmask_buffer[i];\n+\t\t\tunsigned int sec_hitmask = sec_hitmask_buffer[i];\n+#endif\n+\t\t\twhile (prim_hitmask) {\n \t\t\t\tuint32_t hit_index =\n-\t\t\t\t\t\trte_ctz32(prim_hitmask[i])\n+\t\t\t\t\t\trte_ctz32(prim_hitmask)\n \t\t\t\t\t\t>> hitmask_padding;\n \t\t\t\tuint32_t key_idx =\n \t\t\t\trte_atomic_load_explicit(\n@@ -2200,12 +2304,12 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \t\t\t\t\tpositions[i] = key_idx - 1;\n \t\t\t\t\tgoto next_key;\n \t\t\t\t}\n-\t\t\t\tprim_hitmask[i] &= ~(1 << (hit_index << hitmask_padding));\n+\t\t\t\tprim_hitmask &= ~(1 << (hit_index << hitmask_padding));\n \t\t\t}\n \n-\t\t\twhile (sec_hitmask[i]) {\n+\t\t\twhile (sec_hitmask) {\n \t\t\t\tuint32_t hit_index =\n-\t\t\t\t\t\trte_ctz32(sec_hitmask[i])\n+\t\t\t\t\t\trte_ctz32(sec_hitmask)\n \t\t\t\t\t\t>> hitmask_padding;\n \t\t\t\tuint32_t key_idx =\n \t\t\t\trte_atomic_load_explicit(\n@@ -2233,7 +2337,7 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \t\t\t\t\tpositions[i] = key_idx - 1;\n \t\t\t\t\tgoto next_key;\n \t\t\t\t}\n-\t\t\t\tsec_hitmask[i] &= ~(1 << (hit_index << hitmask_padding));\n+\t\t\t\tsec_hitmask &= ~(1 << (hit_index << hitmask_padding));\n \t\t\t}\n next_key:\n \t\t\tcontinue;\ndiff --git a/lib/hash/rte_cuckoo_hash.h b/lib/hash/rte_cuckoo_hash.h\nindex 8ea793c66e..ed18e1f41e 100644\n--- a/lib/hash/rte_cuckoo_hash.h\n+++ b/lib/hash/rte_cuckoo_hash.h\n@@ -137,6 +137,7 @@ enum rte_hash_sig_compare_function {\n \tRTE_HASH_COMPARE_SCALAR = 0,\n \tRTE_HASH_COMPARE_SSE,\n \tRTE_HASH_COMPARE_NEON,\n+\tRTE_HASH_COMPARE_SVE,\n \tRTE_HASH_COMPARE_NUM\n };\n \n", "prefixes": [ "v5", "4/4" ] }