From patchwork Tue Feb 12 07:01:04 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ruifeng Wang X-Patchwork-Id: 50267 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 6CEFF1B43E; Tue, 12 Feb 2019 08:02:25 +0100 (CET) Received: from foss.arm.com (foss.arm.com [217.140.101.70]) by dpdk.org (Postfix) with ESMTP id C08381B3B9 for ; Tue, 12 Feb 2019 08:02:23 +0100 (CET) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D406780D; Mon, 11 Feb 2019 23:02:22 -0800 (PST) Received: from net-arm-thunderx2-02.shanghai.arm.com (unknown [10.169.41.68]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 8ACAE3F589; Mon, 11 Feb 2019 23:02:21 -0800 (PST) From: Ruifeng Wang To: yipeng1.wang@intel.com, jerinj@marvell.com Cc: dev@dpdk.org, honnappa.nagarahalli@arm.com, gavin.hu@arm.com, nd@arm.com, Ruifeng Wang Date: Tue, 12 Feb 2019 15:01:04 +0800 Message-Id: <20190212070104.33830-1-ruifeng.wang@arm.com> X-Mailer: git-send-email 2.17.1 Subject: [dpdk-dev] [PATCH v2] hash: optimize signature compare by using neon intrinsic X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Implemented signature compare function based on neon intrinsic. Hash bulk lookup had 3% - 6% performance gain after optimization. Signed-off-by: Ruifeng Wang Reviewed-by: Gavin Hu Acked-by: Yipeng Wang Acked-by: Jerin Jacob --- v2: * Use 'rte_vect.h' instead of 'arm_neon.h'. * Moved down variable declaration to avoid extra #ifdef. lib/librte_hash/rte_cuckoo_hash.c | 29 ++++++++++++++++++++++++++++- lib/librte_hash/rte_cuckoo_hash.h | 1 + 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c index c01489ba5..0dddce226 100644 --- a/lib/librte_hash/rte_cuckoo_hash.c +++ b/lib/librte_hash/rte_cuckoo_hash.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "rte_hash.h" #include "rte_cuckoo_hash.h" @@ -407,6 +408,10 @@ rte_hash_create(const struct rte_hash_parameters *params) if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE2)) h->sig_cmp_fn = RTE_HASH_COMPARE_SSE; else +#elif defined(RTE_ARCH_ARM64) + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) + h->sig_cmp_fn = RTE_HASH_COMPARE_NEON; + else #endif h->sig_cmp_fn = RTE_HASH_COMPARE_SCALAR; @@ -1581,7 +1586,7 @@ compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, /* For match mask the first bit of every two bits indicates the match */ switch (sig_cmp_fn) { -#ifdef RTE_MACHINE_CPUFLAG_SSE2 +#if defined(RTE_MACHINE_CPUFLAG_SSE2) case RTE_HASH_COMPARE_SSE: /* Compare all signatures in the bucket */ *prim_hash_matches = _mm_movemask_epi8(_mm_cmpeq_epi16( @@ -1594,6 +1599,28 @@ compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, (__m128i const *)sec_bkt->sig_current), _mm_set1_epi16(sig))); break; +#elif defined(RTE_MACHINE_CPUFLAG_NEON) + case RTE_HASH_COMPARE_NEON: { + uint16x8_t vmat, vsig, x; + uint64x2_t x64; + int16x8_t shift = {-15, -13, -11, -9, -7, -5, -3, -1}; + + vsig = vld1q_dup_u16((uint16_t const *)&sig); + /* Compare all signatures in the primary bucket */ + vmat = vceqq_u16(vsig, + vld1q_u16((uint16_t const *)prim_bkt->sig_current)); + x = vshlq_u16(vandq_u16(vmat, vdupq_n_u16(0x8000)), shift); + x64 = vpaddlq_u32(vpaddlq_u16(x)); + *prim_hash_matches = (uint32_t)(vgetq_lane_u64(x64, 0) + + vgetq_lane_u64(x64, 1)); + /* Compare all signatures in the secondary bucket */ + vmat = vceqq_u16(vsig, + vld1q_u16((uint16_t const *)sec_bkt->sig_current)); + x = vshlq_u16(vandq_u16(vmat, vdupq_n_u16(0x8000)), shift); + x64 = vpaddlq_u32(vpaddlq_u16(x)); + *sec_hash_matches = (uint32_t)(vgetq_lane_u64(x64, 0) + + vgetq_lane_u64(x64, 1)); } + break; #endif default: for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { diff --git a/lib/librte_hash/rte_cuckoo_hash.h b/lib/librte_hash/rte_cuckoo_hash.h index eacdaa8d4..0548c97f0 100644 --- a/lib/librte_hash/rte_cuckoo_hash.h +++ b/lib/librte_hash/rte_cuckoo_hash.h @@ -141,6 +141,7 @@ struct rte_hash_key { enum rte_hash_sig_compare_function { RTE_HASH_COMPARE_SCALAR = 0, RTE_HASH_COMPARE_SSE, + RTE_HASH_COMPARE_NEON, RTE_HASH_COMPARE_NUM };