From patchwork Fri Sep 2 09:18:31 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavan Nikhilesh Bhagavatula X-Patchwork-Id: 115790 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id DB0DCA0544; Fri, 2 Sep 2022 11:19:05 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id A3C4642829; Fri, 2 Sep 2022 11:18:58 +0200 (CEST) Received: from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com [67.231.156.173]) by mails.dpdk.org (Postfix) with ESMTP id 2ABC4427F7 for ; Fri, 2 Sep 2022 11:18:56 +0200 (CEST) Received: from pps.filterd (m0045851.ppops.net [127.0.0.1]) by mx0b-0016f401.pphosted.com (8.17.1.5/8.17.1.5) with ESMTP id 281M67kJ016108; Fri, 2 Sep 2022 02:18:52 -0700 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h=from : to : cc : subject : date : message-id : in-reply-to : references : mime-version : content-transfer-encoding : content-type; s=pfpt0220; bh=zXy8hLNoSmbKAQmw+zOVNHbbYr3+loUjRyU9FJDpH+8=; b=F22TU2ccdYs9t6aDpcNzGt/vSOjP5S2F1c2J2qOkOsNMnVXekMu2f7ZtsVrUaqiBoQ7p F3KDwZjOCNjBZHtTzhbyZHh/L6PpjX3fZURHJ7XL+zu/D4lq84cgehw6dPMC53GhTvZR zz/9RsKlNOzPaQrHIztEfLwA0tzUCSC3gAzFu4Xl6X//kX89+cQl0+vdz/TbWMiWaGjf LX5jK6Hup7TLhLuGqJyvA+z4Ejfn9i9FDpVaFoUe36Z6TzV+WRBuHuyyDQNxiGHRE5a5 Xgof+abZwY3zGoNtJz7fy6YxI9lymH+6bcd6NHkHdZB3J3F/8koddqzSgtF82x9k7Lsp 9w== Received: from dc5-exch01.marvell.com ([199.233.59.181]) by mx0b-0016f401.pphosted.com (PPS) with ESMTPS id 3jax17uupt-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT); Fri, 02 Sep 2022 02:18:52 -0700 Received: from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH01.marvell.com (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.2; Fri, 2 Sep 2022 02:18:50 -0700 Received: from maili.marvell.com (10.69.176.80) by DC5-EXCH02.marvell.com (10.69.176.39) with Microsoft SMTP Server id 15.0.1497.18 via Frontend Transport; Fri, 2 Sep 2022 02:18:50 -0700 Received: from MININT-80QBFE8.corp.innovium.com (unknown [10.28.161.88]) by maili.marvell.com (Postfix) with ESMTP id 51E025B6935; Fri, 2 Sep 2022 02:18:48 -0700 (PDT) From: To: , David Christensen , "Ruifeng Wang" , Bruce Richardson , Konstantin Ananyev CC: , Pavan Nikhilesh Subject: [PATCH v2 3/5] examples/l3fwd: use lpm vector path for event vector Date: Fri, 2 Sep 2022 14:48:31 +0530 Message-ID: <20220902091833.9074-3-pbhagavatula@marvell.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220902091833.9074-1-pbhagavatula@marvell.com> References: <20220829094442.3422-1-pbhagavatula@marvell.com> <20220902091833.9074-1-pbhagavatula@marvell.com> MIME-Version: 1.0 X-Proofpoint-GUID: pt1gaD54_zN9XkHa5po--grbjKh4FYP2 X-Proofpoint-ORIG-GUID: pt1gaD54_zN9XkHa5po--grbjKh4FYP2 X-Proofpoint-Virus-Version: vendor=baseguard engine=ICAP:2.0.205,Aquarius:18.0.895,Hydra:6.0.517,FMLib:17.11.122.1 definitions=2022-09-01_12,2022-08-31_03,2022-06-22_01 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Pavan Nikhilesh Use lpm vector path to process event vector. Signed-off-by: Pavan Nikhilesh --- examples/l3fwd/l3fwd_altivec.h | 29 ++++++++++++++ examples/l3fwd/l3fwd_event.h | 71 ++++++++++++++++++++++++++++++++++ examples/l3fwd/l3fwd_lpm.c | 38 ++++++++++-------- examples/l3fwd/l3fwd_neon.h | 45 +++++++++++++++++++++ examples/l3fwd/l3fwd_sse.h | 44 +++++++++++++++++++++ 5 files changed, 211 insertions(+), 16 deletions(-) diff --git a/examples/l3fwd/l3fwd_altivec.h b/examples/l3fwd/l3fwd_altivec.h index 87018f5dbe..e45e138e59 100644 --- a/examples/l3fwd/l3fwd_altivec.h +++ b/examples/l3fwd/l3fwd_altivec.h @@ -222,4 +222,33 @@ send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, } } +static __rte_always_inline uint16_t +process_dst_port(uint16_t *dst_ports, uint16_t nb_elem) +{ + uint16_t i = 0, res; + + while (nb_elem > 7) { + __vector unsigned short dp1; + __vector unsigned short dp; + + dp = (__vector unsigned short)vec_splats((short)dst_ports[0]); + dp1 = *((__vector unsigned short *)&dst_ports[i]); + res = vec_all_eq(dp1, dp); + if (!res) + return BAD_PORT; + + nb_elem -= 8; + i += 8; + } + + while (nb_elem) { + if (dst_ports[i] != dst_ports[0]) + return BAD_PORT; + nb_elem--; + i++; + } + + return dst_ports[0]; +} + #endif /* _L3FWD_ALTIVEC_H_ */ diff --git a/examples/l3fwd/l3fwd_event.h b/examples/l3fwd/l3fwd_event.h index b93841a16f..3fe38aada0 100644 --- a/examples/l3fwd/l3fwd_event.h +++ b/examples/l3fwd/l3fwd_event.h @@ -82,6 +82,27 @@ struct l3fwd_event_resources { uint64_t vector_tmo_ns; }; +#if defined(RTE_ARCH_X86) +#include "l3fwd_sse.h" +#elif defined __ARM_NEON +#include "l3fwd_neon.h" +#elif defined(RTE_ARCH_PPC_64) +#include "l3fwd_altivec.h" +#else +static inline uint16_t +process_dst_port(uint16_t *dst_ports, uint16_t nb_elem) +{ + int i; + + for (i = 0; i < nb_elem; i++) { + if (dst_ports[i] != dst_ports[0]) + return BAD_PORT; + } + + return dst_ports[0]; +} +#endif + static inline void event_vector_attr_validate(struct rte_event_vector *vec, struct rte_mbuf *mbuf) { @@ -103,7 +124,57 @@ event_vector_txq_set(struct rte_event_vector *vec, uint16_t txq) } } +static inline uint16_t +filter_bad_packets(struct rte_mbuf **mbufs, uint16_t *dst_port, + uint16_t nb_pkts) +{ + uint16_t *des_pos, free = 0; + struct rte_mbuf **pos; + int i; + + /* Filter out and free bad packets */ + for (i = 0; i < nb_pkts; i++) { + if (dst_port[i] == BAD_PORT) { + rte_pktmbuf_free(mbufs[i]); + if (!free) { + pos = &mbufs[i]; + des_pos = &dst_port[i]; + } + free++; + continue; + } + + if (free) { + *pos = mbufs[i]; + pos++; + *des_pos = dst_port[i]; + des_pos++; + } + } + return nb_pkts - free; +} + +static inline void +process_event_vector(struct rte_event_vector *vec, uint16_t *dst_port) +{ + uint16_t port, i; + + vec->nb_elem = filter_bad_packets(vec->mbufs, dst_port, vec->nb_elem); + /* Verify destination array */ + port = process_dst_port(dst_port, vec->nb_elem); + if (port == BAD_PORT) { + vec->attr_valid = 0; + for (i = 0; i < vec->nb_elem; i++) { + vec->mbufs[i]->port = dst_port[i]; + rte_event_eth_tx_adapter_txq_set(vec->mbufs[i], 0); + } + } else { + vec->attr_valid = 1; + vec->port = port; + vec->queue = 0; + } +} struct l3fwd_event_resources *l3fwd_get_eventdev_rsrc(void); void l3fwd_event_resource_setup(struct rte_eth_conf *port_conf); diff --git a/examples/l3fwd/l3fwd_lpm.c b/examples/l3fwd/l3fwd_lpm.c index d1b850dd5b..1652b7c470 100644 --- a/examples/l3fwd/l3fwd_lpm.c +++ b/examples/l3fwd/l3fwd_lpm.c @@ -425,24 +425,27 @@ lpm_event_main_loop_tx_q_burst(__rte_unused void *dummy) } static __rte_always_inline void -lpm_process_event_vector(struct rte_event_vector *vec, struct lcore_conf *lconf) +lpm_process_event_vector(struct rte_event_vector *vec, struct lcore_conf *lconf, + uint16_t *dst_port) { struct rte_mbuf **mbufs = vec->mbufs; int i; - /* Process first packet to init vector attributes */ - lpm_process_event_pkt(lconf, mbufs[0]); +#if defined RTE_ARCH_X86 || defined __ARM_NEON || defined RTE_ARCH_PPC_64 if (vec->attr_valid) { - if (mbufs[0]->port != BAD_PORT) - vec->port = mbufs[0]->port; - else - vec->attr_valid = 0; + l3fwd_lpm_process_packets(vec->nb_elem, mbufs, vec->port, + dst_port, lconf, 1); + } else { + for (i = 0; i < vec->nb_elem; i++) + l3fwd_lpm_process_packets(1, &mbufs[i], mbufs[i]->port, + &dst_port[i], lconf, 1); } +#else + for (i = 0; i < vec->nb_elem; i++) + dst_port[i] = lpm_process_event_pkt(lconf, mbufs[i]); +#endif - for (i = 1; i < vec->nb_elem; i++) { - lpm_process_event_pkt(lconf, mbufs[i]); - event_vector_attr_validate(vec, mbufs[i]); - } + process_event_vector(vec, dst_port); } /* Same eventdev loop for single and burst of vector */ @@ -458,6 +461,7 @@ lpm_event_loop_vector(struct l3fwd_event_resources *evt_rsrc, struct rte_event events[MAX_PKT_BURST]; int i, nb_enq = 0, nb_deq = 0; struct lcore_conf *lconf; + uint16_t *dst_port_list; unsigned int lcore_id; if (event_p_id < 0) @@ -465,7 +469,11 @@ lpm_event_loop_vector(struct l3fwd_event_resources *evt_rsrc, lcore_id = rte_lcore_id(); lconf = &lcore_conf[lcore_id]; - + dst_port_list = + rte_zmalloc("", sizeof(uint16_t) * evt_rsrc->vector_size, + RTE_CACHE_LINE_SIZE); + if (dst_port_list == NULL) + return; RTE_LOG(INFO, L3FWD, "entering %s on lcore %u\n", __func__, lcore_id); while (!force_quit) { @@ -483,10 +491,8 @@ lpm_event_loop_vector(struct l3fwd_event_resources *evt_rsrc, events[i].op = RTE_EVENT_OP_FORWARD; } - lpm_process_event_vector(events[i].vec, lconf); - - if (flags & L3FWD_EVENT_TX_DIRECT) - event_vector_txq_set(events[i].vec, 0); + lpm_process_event_vector(events[i].vec, lconf, + dst_port_list); } if (flags & L3FWD_EVENT_TX_ENQ) { diff --git a/examples/l3fwd/l3fwd_neon.h b/examples/l3fwd/l3fwd_neon.h index ce515e0bc4..4d98288707 100644 --- a/examples/l3fwd/l3fwd_neon.h +++ b/examples/l3fwd/l3fwd_neon.h @@ -194,4 +194,49 @@ send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, } } +static __rte_always_inline uint16_t +process_dst_port(uint16_t *dst_ports, uint16_t nb_elem) +{ + uint16_t i = 0, res; + +#if defined(RTE_ARCH_ARM64) + while (nb_elem > 7) { + uint16x8_t dp = vdupq_n_u16(dst_ports[0]); + uint16x8_t dp1; + + dp1 = vld1q_u16(&dst_ports[i]); + dp1 = vceqq_u16(dp1, dp); + res = vminvq_u16(dp1); + if (!res) + return BAD_PORT; + + nb_elem -= 8; + i += 8; + } + + while (nb_elem > 3) { + uint16x4_t dp = vdup_n_u16(dst_ports[0]); + uint16x4_t dp1; + + dp1 = vld1_u16(&dst_ports[i]); + dp1 = vceq_u16(dp1, dp); + res = vminv_u16(dp1); + if (!res) + return BAD_PORT; + + nb_elem -= 4; + i += 4; + } +#endif + + while (nb_elem) { + if (dst_ports[i] != dst_ports[0]) + return BAD_PORT; + nb_elem--; + i++; + } + + return dst_ports[0]; +} + #endif /* _L3FWD_NEON_H_ */ diff --git a/examples/l3fwd/l3fwd_sse.h b/examples/l3fwd/l3fwd_sse.h index 0f0d0323a2..083729cdef 100644 --- a/examples/l3fwd/l3fwd_sse.h +++ b/examples/l3fwd/l3fwd_sse.h @@ -194,4 +194,48 @@ send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, } } +static __rte_always_inline uint16_t +process_dst_port(uint16_t *dst_ports, uint16_t nb_elem) +{ + uint16_t i = 0, res; + + while (nb_elem > 7) { + __m128i dp = _mm_set1_epi16(dst_ports[0]); + __m128i dp1; + + dp1 = _mm_loadu_si128((__m128i *)&dst_ports[i]); + dp1 = _mm_cmpeq_epi16(dp1, dp); + res = _mm_movemask_epi8(dp1); + if (res != 0xFFFF) + return BAD_PORT; + + nb_elem -= 8; + i += 8; + } + + while (nb_elem > 3) { + __m128i dp = _mm_set1_epi16(dst_ports[0]); + __m128i dp1; + + dp1 = _mm_loadu_si128((__m128i *)&dst_ports[i]); + dp1 = _mm_cmpeq_epi16(dp1, dp); + dp1 = _mm_unpacklo_epi16(dp1, dp1); + res = _mm_movemask_ps((__m128)dp1); + if (res != 0xF) + return BAD_PORT; + + nb_elem -= 4; + i += 4; + } + + while (nb_elem) { + if (dst_ports[i] != dst_ports[0]) + return BAD_PORT; + nb_elem--; + i++; + } + + return dst_ports[0]; +} + #endif /* _L3FWD_SSE_H_ */