Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 133166,
    "url": "http://patches.dpdk.org/api/patches/133166/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/20231020165159.1649282-5-yoan.picchi@arm.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20231020165159.1649282-5-yoan.picchi@arm.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20231020165159.1649282-5-yoan.picchi@arm.com",
    "date": "2023-10-20T16:51:59",
    "name": "[v2,4/4] hash: add SVE support for bulk key lookup",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "69237019c69f4333c0de81b2448f3dd9b4cf0c38",
    "submitter": {
        "id": 3196,
        "url": "http://patches.dpdk.org/api/people/3196/?format=api",
        "name": "Yoan Picchi",
        "email": "yoan.picchi@arm.com"
    },
    "delegate": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/users/1/?format=api",
        "username": "tmonjalo",
        "first_name": "Thomas",
        "last_name": "Monjalon",
        "email": "thomas@monjalon.net"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/20231020165159.1649282-5-yoan.picchi@arm.com/mbox/",
    "series": [
        {
            "id": 29947,
            "url": "http://patches.dpdk.org/api/series/29947/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=29947",
            "date": "2023-10-20T16:51:55",
            "name": "hash: add SVE support for bulk key lookup",
            "version": 2,
            "mbox": "http://patches.dpdk.org/series/29947/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/133166/comments/",
    "check": "success",
    "checks": "http://patches.dpdk.org/api/patches/133166/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 76FF5431E0;\n\tMon, 23 Oct 2023 10:55:29 +0200 (CEST)",
            "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 9506D40E96;\n\tMon, 23 Oct 2023 10:55:08 +0200 (CEST)",
            "from foss.arm.com (foss.arm.com [217.140.110.172])\n by mails.dpdk.org (Postfix) with ESMTP id E4BAF402E6\n for <dev@dpdk.org>; Fri, 20 Oct 2023 18:53:02 +0200 (CEST)",
            "from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14])\n by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 5EBBC1595;\n Fri, 20 Oct 2023 09:53:43 -0700 (PDT)",
            "from ampere-altra-2-2.usa.Arm.com (unknown [10.118.91.160])\n by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 6523E3F5A1;\n Fri, 20 Oct 2023 09:53:02 -0700 (PDT)"
        ],
        "From": "Yoan Picchi <yoan.picchi@arm.com>",
        "To": "Yipeng Wang <yipeng1.wang@intel.com>,\n Sameh Gobriel <sameh.gobriel@intel.com>,\n Bruce Richardson <bruce.richardson@intel.com>,\n Vladimir Medvedkin <vladimir.medvedkin@intel.com>",
        "Cc": "dev@dpdk.org, Yoan Picchi <yoan.picchi@arm.com>,\n Harjot Singh <harjot.singh@arm.com>",
        "Subject": "[PATCH v2 4/4] hash: add SVE support for bulk key lookup",
        "Date": "Fri, 20 Oct 2023 16:51:59 +0000",
        "Message-Id": "<20231020165159.1649282-5-yoan.picchi@arm.com>",
        "X-Mailer": "git-send-email 2.25.1",
        "In-Reply-To": "<20231020165159.1649282-1-yoan.picchi@arm.com>",
        "References": "<20231020165159.1649282-1-yoan.picchi@arm.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "X-Mailman-Approved-At": "Mon, 23 Oct 2023 10:55:02 +0200",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org"
    },
    "content": "- Implemented SVE code for comparing signatures in bulk lookup.\n- Added Defines in code for SVE code support.\n- Optimise NEON code\n- New SVE code is ~3% slower than optimized NEON for N2 processor.\n\nSigned-off-by: Yoan Picchi <yoan.picchi@arm.com>\nSigned-off-by: Harjot Singh <harjot.singh@arm.com>\n---\n lib/hash/rte_cuckoo_hash.c | 196 ++++++++++++++++++++++++++++---------\n lib/hash/rte_cuckoo_hash.h |   1 +\n 2 files changed, 151 insertions(+), 46 deletions(-)",
    "diff": "diff --git a/lib/hash/rte_cuckoo_hash.c b/lib/hash/rte_cuckoo_hash.c\nindex a4b907c45c..cda39d1441 100644\n--- a/lib/hash/rte_cuckoo_hash.c\n+++ b/lib/hash/rte_cuckoo_hash.c\n@@ -435,8 +435,11 @@ rte_hash_create(const struct rte_hash_parameters *params)\n \t\th->sig_cmp_fn = RTE_HASH_COMPARE_SSE;\n \telse\n #elif defined(RTE_ARCH_ARM64)\n-\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON))\n+\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {\n \t\th->sig_cmp_fn = RTE_HASH_COMPARE_NEON;\n+\t\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SVE))\n+\t\t\th->sig_cmp_fn = RTE_HASH_COMPARE_SVE;\n+\t}\n \telse\n #endif\n \t\th->sig_cmp_fn = RTE_HASH_COMPARE_SCALAR;\n@@ -1853,37 +1856,103 @@ rte_hash_free_key_with_position(const struct rte_hash *h,\n #if defined(__ARM_NEON)\n \n static inline void\n-compare_signatures_dense(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches,\n-\t\t\tconst struct rte_hash_bucket *prim_bkt,\n-\t\t\tconst struct rte_hash_bucket *sec_bkt,\n+compare_signatures_dense(uint16_t *hitmask_buffer,\n+\t\t\tconst uint16_t *prim_bucket_sigs,\n+\t\t\tconst uint16_t *sec_bucket_sigs,\n \t\t\tuint16_t sig,\n \t\t\tenum rte_hash_sig_compare_function sig_cmp_fn)\n {\n \tunsigned int i;\n \n+\tstatic_assert(sizeof(*hitmask_buffer) >= 2*(RTE_HASH_BUCKET_ENTRIES/8),\n+\t\"The hitmask must be exactly wide enough to accept the whole hitmask if it is dense\");\n+\n \t/* For match mask every bits indicates the match */\n \tswitch (sig_cmp_fn) {\n+#if defined(__ARM_NEON) && RTE_HASH_BUCKET_ENTRIES <= 8\n \tcase RTE_HASH_COMPARE_NEON: {\n-\t\tuint16x8_t vmat, x;\n+\t\tuint16x8_t vmat, hit1, hit2;\n \t\tconst uint16x8_t mask = {0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};\n \t\tconst uint16x8_t vsig = vld1q_dup_u16((uint16_t const *)&sig);\n \n \t\t/* Compare all signatures in the primary bucket */\n-\t\tvmat = vceqq_u16(vsig, vld1q_u16((uint16_t const *)prim_bkt->sig_current));\n-\t\tx = vandq_u16(vmat, mask);\n-\t\t*prim_hash_matches = (uint32_t)(vaddvq_u16(x));\n+\t\tvmat = vceqq_u16(vsig, vld1q_u16(prim_bucket_sigs));\n+\t\thit1 = vandq_u16(vmat, mask);\n+\n \t\t/* Compare all signatures in the secondary bucket */\n-\t\tvmat = vceqq_u16(vsig, vld1q_u16((uint16_t const *)sec_bkt->sig_current));\n-\t\tx = vandq_u16(vmat, mask);\n-\t\t*sec_hash_matches = (uint32_t)(vaddvq_u16(x));\n+\t\tvmat = vceqq_u16(vsig, vld1q_u16(sec_bucket_sigs));\n+\t\thit2 = vandq_u16(vmat, mask);\n+\n+\t\thit2 = vshlq_n_u16(hit2, RTE_HASH_BUCKET_ENTRIES);\n+\t\thit2 = vorrq_u16(hit1, hit2);\n+\t\t*hitmask_buffer = vaddvq_u16(hit2);\n+\t\t}\n+\t\tbreak;\n+#endif\n+#if defined(RTE_HAS_SVE_ACLE)\n+\tcase RTE_HASH_COMPARE_SVE: {\n+\t\tsvuint16_t vsign, shift, sv_matches;\n+\t\tsvbool_t pred, match, bucket_wide_pred;\n+\t\tint i = 0;\n+\t\tuint64_t vl = svcnth();\n+\n+\t\tvsign = svdup_u16(sig);\n+\t\tshift = svindex_u16(0, 1);\n+\n+\t\tif (vl >= 2 * RTE_HASH_BUCKET_ENTRIES && RTE_HASH_BUCKET_ENTRIES <= 8) {\n+\t\t\tsvuint16_t primary_array_vect, secondary_array_vect;\n+\t\t\tbucket_wide_pred = svwhilelt_b16(0, RTE_HASH_BUCKET_ENTRIES);\n+\t\t\tprimary_array_vect = svld1_u16(bucket_wide_pred, prim_bucket_sigs);\n+\t\t\tsecondary_array_vect = svld1_u16(bucket_wide_pred, sec_bucket_sigs);\n+\n+\t\t\t/* We merged the two vectors so we can do both comparison at once */\n+\t\t\tprimary_array_vect = svsplice_u16(bucket_wide_pred,\n+\t\t\t\tprimary_array_vect,\n+\t\t\t\tsecondary_array_vect);\n+\t\t\tpred = svwhilelt_b16(0, 2*RTE_HASH_BUCKET_ENTRIES);\n+\n+\t\t\t/* Compare all signatures in the buckets */\n+\t\t\tmatch = svcmpeq_u16(pred, vsign, primary_array_vect);\n+\t\t\tif (svptest_any(svptrue_b16(), match)) {\n+\t\t\t\tsv_matches = svdup_u16(1);\n+\t\t\t\tsv_matches = svlsl_u16_z(match, sv_matches, shift);\n+\t\t\t\t*hitmask_buffer = svorv_u16(svptrue_b16(), sv_matches);\n+\t\t\t}\n+\t\t} else {\n+\t\t\tdo {\n+\t\t\t\tpred = svwhilelt_b16(i, RTE_HASH_BUCKET_ENTRIES);\n+\t\t\t\tint lower_half = 0;\n+\t\t\t\tint upper_half = 0;\n+\t\t\t\t/* Compare all signatures in the primary bucket */\n+\t\t\t\tmatch = svcmpeq_u16(pred, vsign, svld1_u16(pred,\n+\t\t\t\t\t\t\t&prim_bucket_sigs[i]));\n+\t\t\t\tif (svptest_any(svptrue_b16(), match)) {\n+\t\t\t\t\tsv_matches = svdup_u16(1);\n+\t\t\t\t\tsv_matches = svlsl_u16_z(match, sv_matches, shift);\n+\t\t\t\t\tlower_half = svorv_u16(svptrue_b16(), sv_matches);\n+\t\t\t\t}\n+\t\t\t\t/* Compare all signatures in the secondary bucket */\n+\t\t\t\tmatch = svcmpeq_u16(pred, vsign, svld1_u16(pred,\n+\t\t\t\t\t\t\t&sec_bucket_sigs[i]));\n+\t\t\t\tif (svptest_any(svptrue_b16(), match)) {\n+\t\t\t\t\tsv_matches = svdup_u16(1);\n+\t\t\t\t\tsv_matches = svlsl_u16_z(match, sv_matches, shift);\n+\t\t\t\t\tupper_half = svorv_u16(svptrue_b16(), sv_matches)\n+\t\t\t\t\t\t<< RTE_HASH_BUCKET_ENTRIES;\n+\t\t\t\t}\n+\t\t\t\t*(hitmask_buffer+(i/8)) = lower_half | upper_half;\n+\t\t\t\ti += vl;\n+\t\t\t} while (i < RTE_HASH_BUCKET_ENTRIES);\n+\t\t}\n \t\t}\n \t\tbreak;\n+#endif\n \tdefault:\n \t\tfor (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {\n-\t\t\t*prim_hash_matches |=\n-\t\t\t\t((sig == prim_bkt->sig_current[i]) << i);\n-\t\t\t*sec_hash_matches |=\n-\t\t\t\t((sig == sec_bkt->sig_current[i]) << i);\n+\t\t\t*hitmask_buffer |=\n+\t\t\t\t((sig == prim_bucket_sigs[i]) << i);\n+\t\t\t*hitmask_buffer |=\n+\t\t\t\t((sig == sec_bucket_sigs[i]) << i) << RTE_HASH_BUCKET_ENTRIES;\n \t\t}\n \t}\n }\n@@ -1901,7 +1970,7 @@ compare_signatures_sparse(uint32_t *prim_hash_matches, uint32_t *sec_hash_matche\n \n \t/* For match mask the first bit of every two bits indicates the match */\n \tswitch (sig_cmp_fn) {\n-#if defined(__SSE2__)\n+#if defined(__SSE2__) && RTE_HASH_BUCKET_ENTRIES <= 8\n \tcase RTE_HASH_COMPARE_SSE:\n \t\t/* Compare all signatures in the bucket */\n \t\t*prim_hash_matches = _mm_movemask_epi8(_mm_cmpeq_epi16(\n@@ -1941,14 +2010,18 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \tuint64_t hits = 0;\n \tint32_t i;\n \tint32_t ret;\n-\tuint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n-\tuint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n \tstruct rte_hash_bucket *cur_bkt, *next_bkt;\n \n #if defined(__ARM_NEON)\n \tconst int hitmask_padding = 0;\n+\tuint16_t hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n+\n+\tstatic_assert(sizeof(*hitmask_buffer)*8/2 == RTE_HASH_BUCKET_ENTRIES,\n+\t\"The hitmask must be exactly wide enough to accept the whole hitmask when it is dense\");\n #else\n \tconst int hitmask_padding = 1;\n+\tuint32_t prim_hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n+\tuint32_t sec_hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n #endif\n \n \t__hash_rw_reader_lock(h);\n@@ -1956,18 +2029,24 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t/* Compare signatures and prefetch key slot of first hit */\n \tfor (i = 0; i < num_keys; i++) {\n #if defined(__ARM_NEON)\n-\t\tcompare_signatures_dense(&prim_hitmask[i], &sec_hitmask[i],\n-\t\t\tprimary_bkt[i], secondary_bkt[i],\n+\t\tuint16_t *hitmask = &hitmask_buffer[i];\n+\t\tcompare_signatures_dense(hitmask,\n+\t\t\tprimary_bkt[i]->sig_current,\n+\t\t\tsecondary_bkt[i]->sig_current,\n \t\t\tsig[i], h->sig_cmp_fn);\n+\t\tconst unsigned int prim_hitmask = *(uint8_t *)(hitmask);\n+\t\tconst unsigned int sec_hitmask = *((uint8_t *)(hitmask)+1);\n #else\n-\t\tcompare_signatures_sparse(&prim_hitmask[i], &sec_hitmask[i],\n+\t\tcompare_signatures_sparse(&prim_hitmask_buffer[i], &sec_hitmask_buffer[i],\n \t\t\tprimary_bkt[i], secondary_bkt[i],\n \t\t\tsig[i], h->sig_cmp_fn);\n+\t\tconst unsigned int prim_hitmask = prim_hitmask_buffer[i];\n+\t\tconst unsigned int sec_hitmask = sec_hitmask_buffer[i];\n #endif\n \n-\t\tif (prim_hitmask[i]) {\n+\t\tif (prim_hitmask) {\n \t\t\tuint32_t first_hit =\n-\t\t\t\t\t__builtin_ctzl(prim_hitmask[i])\n+\t\t\t\t\t__builtin_ctzl(prim_hitmask)\n \t\t\t\t\t>> hitmask_padding;\n \t\t\tuint32_t key_idx =\n \t\t\t\tprimary_bkt[i]->key_idx[first_hit];\n@@ -1979,9 +2058,9 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t\t\tcontinue;\n \t\t}\n \n-\t\tif (sec_hitmask[i]) {\n+\t\tif (sec_hitmask) {\n \t\t\tuint32_t first_hit =\n-\t\t\t\t\t__builtin_ctzl(sec_hitmask[i])\n+\t\t\t\t\t__builtin_ctzl(sec_hitmask)\n \t\t\t\t\t>> hitmask_padding;\n \t\t\tuint32_t key_idx =\n \t\t\t\tsecondary_bkt[i]->key_idx[first_hit];\n@@ -1996,9 +2075,17 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t/* Compare keys, first hits in primary first */\n \tfor (i = 0; i < num_keys; i++) {\n \t\tpositions[i] = -ENOENT;\n-\t\twhile (prim_hitmask[i]) {\n+#if defined(__ARM_NEON)\n+\t\tuint16_t *hitmask = &hitmask_buffer[i];\n+\t\tunsigned int prim_hitmask = *(uint8_t *)(hitmask);\n+\t\tunsigned int sec_hitmask = *((uint8_t *)(hitmask)+1);\n+#else\n+\t\tunsigned int prim_hitmask = prim_hitmask_buffer[i];\n+\t\tunsigned int sec_hitmask = sec_hitmask_buffer[i];\n+#endif\n+\t\twhile (prim_hitmask) {\n \t\t\tuint32_t hit_index =\n-\t\t\t\t\t__builtin_ctzl(prim_hitmask[i])\n+\t\t\t\t\t__builtin_ctzl(prim_hitmask)\n \t\t\t\t\t>> hitmask_padding;\n \t\t\tuint32_t key_idx =\n \t\t\t\tprimary_bkt[i]->key_idx[hit_index];\n@@ -2021,12 +2108,12 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t\t\t\tpositions[i] = key_idx - 1;\n \t\t\t\tgoto next_key;\n \t\t\t}\n-\t\t\tprim_hitmask[i] &= ~(1 << (hit_index << hitmask_padding));\n+\t\t\tprim_hitmask &= ~(1 << (hit_index << hitmask_padding));\n \t\t}\n \n-\t\twhile (sec_hitmask[i]) {\n+\t\twhile (sec_hitmask) {\n \t\t\tuint32_t hit_index =\n-\t\t\t\t\t__builtin_ctzl(sec_hitmask[i])\n+\t\t\t\t\t__builtin_ctzl(sec_hitmask)\n \t\t\t\t\t>> hitmask_padding;\n \t\t\tuint32_t key_idx =\n \t\t\t\tsecondary_bkt[i]->key_idx[hit_index];\n@@ -2050,7 +2137,7 @@ __bulk_lookup_l(const struct rte_hash *h, const void **keys,\n \t\t\t\tpositions[i] = key_idx - 1;\n \t\t\t\tgoto next_key;\n \t\t\t}\n-\t\t\tsec_hitmask[i] &= ~(1 << (hit_index << hitmask_padding));\n+\t\t\tsec_hitmask &= ~(1 << (hit_index << hitmask_padding));\n \t\t}\n next_key:\n \t\tcontinue;\n@@ -2100,15 +2187,18 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \tuint64_t hits = 0;\n \tint32_t i;\n \tint32_t ret;\n-\tuint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n-\tuint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n \tstruct rte_hash_bucket *cur_bkt, *next_bkt;\n \tuint32_t cnt_b, cnt_a;\n \n #if defined(__ARM_NEON)\n \tconst int hitmask_padding = 0;\n+\tuint16_t hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n+\tstatic_assert(sizeof(*hitmask_buffer)*8/2 == RTE_HASH_BUCKET_ENTRIES,\n+\t\"The hitmask must be exactly wide enough to accept the whole hitmask chen it is dense\");\n #else\n \tconst int hitmask_padding = 1;\n+\tuint32_t prim_hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n+\tuint32_t sec_hitmask_buffer[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n #endif\n \n \tfor (i = 0; i < num_keys; i++)\n@@ -2125,18 +2215,24 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \t\t/* Compare signatures and prefetch key slot of first hit */\n \t\tfor (i = 0; i < num_keys; i++) {\n #if defined(__ARM_NEON)\n-\t\t\tcompare_signatures_dense(&prim_hitmask[i], &sec_hitmask[i],\n-\t\t\t\tprimary_bkt[i], secondary_bkt[i],\n+\t\t\tuint16_t *hitmask = &hitmask_buffer[i];\n+\t\t\tcompare_signatures_dense(hitmask,\n+\t\t\t\tprimary_bkt[i]->sig_current,\n+\t\t\t\tsecondary_bkt[i]->sig_current,\n \t\t\t\tsig[i], h->sig_cmp_fn);\n+\t\t\tconst unsigned int prim_hitmask = *(uint8_t *)(hitmask);\n+\t\t\tconst unsigned int sec_hitmask = *((uint8_t *)(hitmask)+1);\n #else\n-\t\t\tcompare_signatures_sparse(&prim_hitmask[i], &sec_hitmask[i],\n+\t\t\tcompare_signatures_sparse(&prim_hitmask_buffer[i], &sec_hitmask_buffer[i],\n \t\t\t\tprimary_bkt[i], secondary_bkt[i],\n \t\t\t\tsig[i], h->sig_cmp_fn);\n+\t\t\tconst unsigned int prim_hitmask = prim_hitmask_buffer[i];\n+\t\t\tconst unsigned int sec_hitmask = sec_hitmask_buffer[i];\n #endif\n \n-\t\t\tif (prim_hitmask[i]) {\n+\t\t\tif (prim_hitmask) {\n \t\t\t\tuint32_t first_hit =\n-\t\t\t\t\t\t__builtin_ctzl(prim_hitmask[i])\n+\t\t\t\t\t\t__builtin_ctzl(prim_hitmask)\n \t\t\t\t\t\t>> hitmask_padding;\n \t\t\t\tuint32_t key_idx =\n \t\t\t\t\tprimary_bkt[i]->key_idx[first_hit];\n@@ -2148,9 +2244,9 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \t\t\t\tcontinue;\n \t\t\t}\n \n-\t\t\tif (sec_hitmask[i]) {\n+\t\t\tif (sec_hitmask) {\n \t\t\t\tuint32_t first_hit =\n-\t\t\t\t\t\t__builtin_ctzl(sec_hitmask[i])\n+\t\t\t\t\t\t__builtin_ctzl(sec_hitmask)\n \t\t\t\t\t\t>> hitmask_padding;\n \t\t\t\tuint32_t key_idx =\n \t\t\t\t\tsecondary_bkt[i]->key_idx[first_hit];\n@@ -2164,9 +2260,17 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \n \t\t/* Compare keys, first hits in primary first */\n \t\tfor (i = 0; i < num_keys; i++) {\n-\t\t\twhile (prim_hitmask[i]) {\n+#if defined(__ARM_NEON)\n+\t\t\tuint16_t *hitmask = &hitmask_buffer[i];\n+\t\t\tunsigned int prim_hitmask = *(uint8_t *)(hitmask);\n+\t\t\tunsigned int sec_hitmask = *((uint8_t *)(hitmask)+1);\n+#else\n+\t\t\tunsigned int prim_hitmask = prim_hitmask_buffer[i];\n+\t\t\tunsigned int sec_hitmask = sec_hitmask_buffer[i];\n+#endif\n+\t\t\twhile (prim_hitmask) {\n \t\t\t\tuint32_t hit_index =\n-\t\t\t\t\t\t__builtin_ctzl(prim_hitmask[i])\n+\t\t\t\t\t\t__builtin_ctzl(prim_hitmask)\n \t\t\t\t\t\t>> hitmask_padding;\n \t\t\t\tuint32_t key_idx =\n \t\t\t\t__atomic_load_n(\n@@ -2193,12 +2297,12 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \t\t\t\t\tpositions[i] = key_idx - 1;\n \t\t\t\t\tgoto next_key;\n \t\t\t\t}\n-\t\t\t\tprim_hitmask[i] &= ~(1 << (hit_index << hitmask_padding));\n+\t\t\t\tprim_hitmask &= ~(1 << (hit_index << hitmask_padding));\n \t\t\t}\n \n-\t\t\twhile (sec_hitmask[i]) {\n+\t\t\twhile (sec_hitmask) {\n \t\t\t\tuint32_t hit_index =\n-\t\t\t\t\t\t__builtin_ctzl(sec_hitmask[i])\n+\t\t\t\t\t\t__builtin_ctzl(sec_hitmask)\n \t\t\t\t\t\t>> hitmask_padding;\n \t\t\t\tuint32_t key_idx =\n \t\t\t\t__atomic_load_n(\n@@ -2226,7 +2330,7 @@ __bulk_lookup_lf(const struct rte_hash *h, const void **keys,\n \t\t\t\t\tpositions[i] = key_idx - 1;\n \t\t\t\t\tgoto next_key;\n \t\t\t\t}\n-\t\t\t\tsec_hitmask[i] &= ~(1 << (hit_index << hitmask_padding));\n+\t\t\t\tsec_hitmask &= ~(1 << (hit_index << hitmask_padding));\n \t\t\t}\n next_key:\n \t\t\tcontinue;\ndiff --git a/lib/hash/rte_cuckoo_hash.h b/lib/hash/rte_cuckoo_hash.h\nindex eb2644f74b..356ec2a69e 100644\n--- a/lib/hash/rte_cuckoo_hash.h\n+++ b/lib/hash/rte_cuckoo_hash.h\n@@ -148,6 +148,7 @@ enum rte_hash_sig_compare_function {\n \tRTE_HASH_COMPARE_SCALAR = 0,\n \tRTE_HASH_COMPARE_SSE,\n \tRTE_HASH_COMPARE_NEON,\n+\tRTE_HASH_COMPARE_SVE,\n \tRTE_HASH_COMPARE_NUM\n };\n \n",
    "prefixes": [
        "v2",
        "4/4"
    ]
}