From patchwork Tue Dec 11 05:55:09 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Qi Zhang X-Patchwork-Id: 48626 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 08CD9322C; Tue, 11 Dec 2018 06:53:57 +0100 (CET) Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by dpdk.org (Postfix) with ESMTP id 31DA32C16 for ; Tue, 11 Dec 2018 06:53:54 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga106.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 10 Dec 2018 21:53:53 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.56,341,1539673200"; d="scan'208";a="301115123" Received: from dpdk51.sh.intel.com ([10.67.110.190]) by fmsmga006.fm.intel.com with ESMTP; 10 Dec 2018 21:53:52 -0800 From: Qi Zhang To: ferruh.yigit@intel.com, bruce.richardson@intel.com, keith.wiles@intel.com, konstantin.ananyev@intel.com Cc: dev@dpdk.org, wenzhuo.lu@intel.com, bernard.iremonger@intel.com, Qi Zhang Date: Tue, 11 Dec 2018 13:55:09 +0800 Message-Id: <20181211055511.32284-2-qi.z.zhang@intel.com> X-Mailer: git-send-email 2.13.6 In-Reply-To: <20181211055511.32284-1-qi.z.zhang@intel.com> References: <20181122172632.6229-1-qi.z.zhang@intel.com> <20181211055511.32284-1-qi.z.zhang@intel.com> Subject: [dpdk-dev] [PATCH v2 1/3] app/testpmd: code refactory for macswap X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Move macswap workload to dedicate function, so we can further enable platform specific optimized version. Signed-off-by: Qi Zhang --- app/test-pmd/macswap.c | 32 ++--------------------------- app/test-pmd/macswap.h | 40 ++++++++++++++++++++++++++++++++++++ app/test-pmd/macswap_common.h | 47 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 30 deletions(-) create mode 100644 app/test-pmd/macswap.h create mode 100644 app/test-pmd/macswap_common.h diff --git a/app/test-pmd/macswap.c b/app/test-pmd/macswap.c index a8384d5b8..849194fe2 100644 --- a/app/test-pmd/macswap.c +++ b/app/test-pmd/macswap.c @@ -66,6 +66,7 @@ #include #include "testpmd.h" +#include "macswap.h" /* * MAC swap forwarding mode: Swap the source and the destination Ethernet @@ -76,15 +77,9 @@ pkt_burst_mac_swap(struct fwd_stream *fs) { struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; struct rte_port *txp; - struct rte_mbuf *mb; - struct ether_hdr *eth_hdr; - struct ether_addr addr; uint16_t nb_rx; uint16_t nb_tx; - uint16_t i; uint32_t retry; - uint64_t ol_flags = 0; - uint64_t tx_offloads; #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES uint64_t start_tsc; uint64_t end_tsc; @@ -108,32 +103,9 @@ pkt_burst_mac_swap(struct fwd_stream *fs) #endif fs->rx_packets += nb_rx; txp = &ports[fs->tx_port]; - tx_offloads = txp->dev_conf.txmode.offloads; - if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) - ol_flags = PKT_TX_VLAN_PKT; - if (tx_offloads & DEV_TX_OFFLOAD_QINQ_INSERT) - ol_flags |= PKT_TX_QINQ_PKT; - if (tx_offloads & DEV_TX_OFFLOAD_MACSEC_INSERT) - ol_flags |= PKT_TX_MACSEC; - for (i = 0; i < nb_rx; i++) { - if (likely(i < nb_rx - 1)) - rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1], - void *)); - mb = pkts_burst[i]; - eth_hdr = rte_pktmbuf_mtod(mb, struct ether_hdr *); - /* Swap dest and src mac addresses. */ - ether_addr_copy(ð_hdr->d_addr, &addr); - ether_addr_copy(ð_hdr->s_addr, ð_hdr->d_addr); - ether_addr_copy(&addr, ð_hdr->s_addr); + do_macswap(pkts_burst, nb_rx, txp); - mb->ol_flags &= IND_ATTACHED_MBUF | EXT_ATTACHED_MBUF; - mb->ol_flags |= ol_flags; - mb->l2_len = sizeof(struct ether_hdr); - mb->l3_len = sizeof(struct ipv4_hdr); - mb->vlan_tci = txp->tx_vlan_id; - mb->vlan_tci_outer = txp->tx_vlan_id_outer; - } nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx); /* * Retry if necessary diff --git a/app/test-pmd/macswap.h b/app/test-pmd/macswap.h new file mode 100644 index 000000000..14e665bd2 --- /dev/null +++ b/app/test-pmd/macswap.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef _MACSWAP_H_ +#define _MACSWAP_H_ + +#include "macswap_common.h" + +static inline void +do_macswap(struct rte_mbuf *pkts[], uint16_t nb, + struct rte_port *txp) +{ + struct ether_hdr *eth_hdr; + struct rte_mbuf *mb; + struct ether_addr addr; + uint64_t ol_flags; + int i; + + ol_flags = ol_flags_init(txp->dev_conf.txmode.offloads); + vlan_qinq_set(pkts, nb, ol_flags, + txp->tx_vlan_id, txp->tx_vlan_id_outer); + + for (i = 0; i < nb; i++) { + if (likely(i < nb - 1)) + rte_prefetch0(rte_pktmbuf_mtod(pkts[i+1], void *)); + mb = pkts[i]; + + eth_hdr = rte_pktmbuf_mtod(mb, struct ether_hdr *); + + /* Swap dest and src mac addresses. */ + ether_addr_copy(ð_hdr->d_addr, &addr); + ether_addr_copy(ð_hdr->s_addr, ð_hdr->d_addr); + ether_addr_copy(&addr, ð_hdr->s_addr); + + mbuf_field_set(mb, ol_flags); + } +} + +#endif /* _BPF_CMD_H_ */ diff --git a/app/test-pmd/macswap_common.h b/app/test-pmd/macswap_common.h new file mode 100644 index 000000000..ab0a5b5ef --- /dev/null +++ b/app/test-pmd/macswap_common.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef _MACSWAP_COMMON_H_ +#define _MACSWAP_COMMON_H_ + +static inline uint64_t +ol_flags_init(uint64_t tx_offload) +{ + uint64_t ol_flags = 0; + + ol_flags |= (tx_offload & DEV_TX_OFFLOAD_VLAN_INSERT) ? + PKT_TX_VLAN : 0; + ol_flags |= (tx_offload & DEV_TX_OFFLOAD_QINQ_INSERT) ? + PKT_TX_QINQ : 0; + ol_flags |= (tx_offload & DEV_TX_OFFLOAD_MACSEC_INSERT) ? + PKT_TX_MACSEC : 0; + + return ol_flags; +} + +static inline void +vlan_qinq_set(struct rte_mbuf *pkts[], uint16_t nb, + uint64_t ol_flags, uint16_t vlan, uint16_t outer_vlan) +{ + int i; + + if (ol_flags & PKT_TX_VLAN) + for (i = 0; i < nb; i++) + pkts[i]->vlan_tci = vlan; + if (ol_flags & PKT_TX_QINQ) + for (i = 0; i < nb; i++) + pkts[i]->vlan_tci_outer = outer_vlan; +} + +static inline void +mbuf_field_set(struct rte_mbuf *mb, uint64_t ol_flags) +{ + mb->ol_flags &= IND_ATTACHED_MBUF | EXT_ATTACHED_MBUF; + mb->ol_flags |= ol_flags; + mb->l2_len = sizeof(struct ether_hdr); + mb->l3_len = sizeof(struct ipv4_hdr); +} + +#endif /* _BPF_CMD_H_ */ + From patchwork Tue Dec 11 05:55:10 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Qi Zhang X-Patchwork-Id: 48627 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id D3DB94C74; Tue, 11 Dec 2018 06:53:58 +0100 (CET) Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by dpdk.org (Postfix) with ESMTP id A91D42C6A for ; Tue, 11 Dec 2018 06:53:55 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga106.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 10 Dec 2018 21:53:55 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.56,341,1539673200"; d="scan'208";a="301115129" Received: from dpdk51.sh.intel.com ([10.67.110.190]) by fmsmga006.fm.intel.com with ESMTP; 10 Dec 2018 21:53:53 -0800 From: Qi Zhang To: ferruh.yigit@intel.com, bruce.richardson@intel.com, keith.wiles@intel.com, konstantin.ananyev@intel.com Cc: dev@dpdk.org, wenzhuo.lu@intel.com, bernard.iremonger@intel.com, Qi Zhang Date: Tue, 11 Dec 2018 13:55:10 +0800 Message-Id: <20181211055511.32284-3-qi.z.zhang@intel.com> X-Mailer: git-send-email 2.13.6 In-Reply-To: <20181211055511.32284-1-qi.z.zhang@intel.com> References: <20181122172632.6229-1-qi.z.zhang@intel.com> <20181211055511.32284-1-qi.z.zhang@intel.com> Subject: [dpdk-dev] [PATCH v2 2/3] app/testpmd: improve MAC swap performance for x86 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" The patch optimizes the mac swap operation by taking advantage of SSE instructions, it only impacts x86 platform. Signed-off-by: Qi Zhang --- app/test-pmd/macswap.c | 4 ++++ app/test-pmd/macswap_common.h | 1 - app/test-pmd/macswap_sse.h | 45 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 app/test-pmd/macswap_sse.h diff --git a/app/test-pmd/macswap.c b/app/test-pmd/macswap.c index 849194fe2..cbb41b728 100644 --- a/app/test-pmd/macswap.c +++ b/app/test-pmd/macswap.c @@ -66,7 +66,11 @@ #include #include "testpmd.h" +#ifdef RTE_ARCH_X86 +#include "macswap_sse.h" +#else #include "macswap.h" +#endif /* * MAC swap forwarding mode: Swap the source and the destination Ethernet diff --git a/app/test-pmd/macswap_common.h b/app/test-pmd/macswap_common.h index ab0a5b5ef..8c5518596 100644 --- a/app/test-pmd/macswap_common.h +++ b/app/test-pmd/macswap_common.h @@ -44,4 +44,3 @@ mbuf_field_set(struct rte_mbuf *mb, uint64_t ol_flags) } #endif /* _BPF_CMD_H_ */ - diff --git a/app/test-pmd/macswap_sse.h b/app/test-pmd/macswap_sse.h new file mode 100644 index 000000000..79f4f9a7c --- /dev/null +++ b/app/test-pmd/macswap_sse.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef _L2FWD_SSE_H_ +#define _L2FWD_SSE_H_ + +#include "macswap_common.h" + +static inline void +do_macswap(struct rte_mbuf *pkts[], uint16_t nb, + struct rte_port *txp) +{ + struct ether_hdr *eth_hdr; + struct rte_mbuf *mb; + uint64_t ol_flags; + int i; + __m128i addr; + __m128i shfl_msk = _mm_set_epi8(15, 14, 13, 12, + 5, 4, 3, 2, + 1, 0, 11, 10, + 9, 8, 7, 6); + + ol_flags = ol_flags_init(txp->dev_conf.txmode.offloads); + vlan_qinq_set(pkts, nb, ol_flags, + txp->tx_vlan_id, txp->tx_vlan_id_outer); + + for (i = 0; i < nb; i++) { + if (likely(i < nb - 1)) + rte_prefetch0(rte_pktmbuf_mtod(pkts[i+1], void *)); + mb = pkts[i]; + + eth_hdr = rte_pktmbuf_mtod(mb, struct ether_hdr *); + + /* Swap dest and src mac addresses. */ + addr = _mm_loadu_si128((__m128i *)eth_hdr); + addr = _mm_shuffle_epi8(addr, shfl_msk); + _mm_storeu_si128((__m128i *)eth_hdr, addr); + + mbuf_field_set(mb, ol_flags); + } +} + +#endif /* _BPF_CMD_H_ */ + From patchwork Tue Dec 11 05:55:11 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Qi Zhang X-Patchwork-Id: 48628 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 5F1B24C90; Tue, 11 Dec 2018 06:54:00 +0100 (CET) Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by dpdk.org (Postfix) with ESMTP id 2A03C3256 for ; Tue, 11 Dec 2018 06:53:57 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga106.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 10 Dec 2018 21:53:56 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.56,341,1539673200"; d="scan'208";a="301115132" Received: from dpdk51.sh.intel.com ([10.67.110.190]) by fmsmga006.fm.intel.com with ESMTP; 10 Dec 2018 21:53:55 -0800 From: Qi Zhang To: ferruh.yigit@intel.com, bruce.richardson@intel.com, keith.wiles@intel.com, konstantin.ananyev@intel.com Cc: dev@dpdk.org, wenzhuo.lu@intel.com, bernard.iremonger@intel.com, Qi Zhang Date: Tue, 11 Dec 2018 13:55:11 +0800 Message-Id: <20181211055511.32284-4-qi.z.zhang@intel.com> X-Mailer: git-send-email 2.13.6 In-Reply-To: <20181211055511.32284-1-qi.z.zhang@intel.com> References: <20181122172632.6229-1-qi.z.zhang@intel.com> <20181211055511.32284-1-qi.z.zhang@intel.com> Subject: [dpdk-dev] [PATCH v2 3/3] app/testpmd: further improve MAC swap performance for x86 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Do four packets macswap in same loop iterate to squeeze more CPU cycles. Signed-off-by: Qi Zhang --- app/test-pmd/macswap_sse.h | 62 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/app/test-pmd/macswap_sse.h b/app/test-pmd/macswap_sse.h index 79f4f9a7c..df2875ace 100644 --- a/app/test-pmd/macswap_sse.h +++ b/app/test-pmd/macswap_sse.h @@ -11,11 +11,12 @@ static inline void do_macswap(struct rte_mbuf *pkts[], uint16_t nb, struct rte_port *txp) { - struct ether_hdr *eth_hdr; - struct rte_mbuf *mb; + struct ether_hdr *eth_hdr[4]; + struct rte_mbuf *mb[4]; uint64_t ol_flags; int i; - __m128i addr; + int r; + __m128i addr0, addr1, addr2, addr3; __m128i shfl_msk = _mm_set_epi8(15, 14, 13, 12, 5, 4, 3, 2, 1, 0, 11, 10, @@ -25,19 +26,56 @@ do_macswap(struct rte_mbuf *pkts[], uint16_t nb, vlan_qinq_set(pkts, nb, ol_flags, txp->tx_vlan_id, txp->tx_vlan_id_outer); - for (i = 0; i < nb; i++) { - if (likely(i < nb - 1)) - rte_prefetch0(rte_pktmbuf_mtod(pkts[i+1], void *)); - mb = pkts[i]; + i = 0; + r = nb; + + while (r >= 4) { + mb[0] = pkts[i++]; + eth_hdr[0] = rte_pktmbuf_mtod(mb[0], struct ether_hdr *); + addr0 = _mm_loadu_si128((__m128i *)eth_hdr[0]); + + mb[1] = pkts[i++]; + eth_hdr[1] = rte_pktmbuf_mtod(mb[1], struct ether_hdr *); + addr1 = _mm_loadu_si128((__m128i *)eth_hdr[1]); + + + mb[2] = pkts[i++]; + eth_hdr[2] = rte_pktmbuf_mtod(mb[2], struct ether_hdr *); + addr2 = _mm_loadu_si128((__m128i *)eth_hdr[2]); + + mb[3] = pkts[i++]; + eth_hdr[3] = rte_pktmbuf_mtod(mb[3], struct ether_hdr *); + addr3 = _mm_loadu_si128((__m128i *)eth_hdr[3]); - eth_hdr = rte_pktmbuf_mtod(mb, struct ether_hdr *); + addr0 = _mm_shuffle_epi8(addr0, shfl_msk); + addr1 = _mm_shuffle_epi8(addr1, shfl_msk); + addr2 = _mm_shuffle_epi8(addr2, shfl_msk); + addr3 = _mm_shuffle_epi8(addr3, shfl_msk); + + _mm_storeu_si128((__m128i *)eth_hdr[0], addr0); + _mm_storeu_si128((__m128i *)eth_hdr[1], addr1); + _mm_storeu_si128((__m128i *)eth_hdr[2], addr2); + _mm_storeu_si128((__m128i *)eth_hdr[3], addr3); + + mbuf_field_set(mb[0], ol_flags); + mbuf_field_set(mb[1], ol_flags); + mbuf_field_set(mb[2], ol_flags); + mbuf_field_set(mb[3], ol_flags); + r -= 4; + } + + for ( ; i < nb; i++) { + if (i < nb - 1) + rte_prefetch0(rte_pktmbuf_mtod(pkts[i+1], void *)); + mb[0] = pkts[i]; + eth_hdr[0] = rte_pktmbuf_mtod(mb[0], struct ether_hdr *); /* Swap dest and src mac addresses. */ - addr = _mm_loadu_si128((__m128i *)eth_hdr); - addr = _mm_shuffle_epi8(addr, shfl_msk); - _mm_storeu_si128((__m128i *)eth_hdr, addr); + addr0 = _mm_loadu_si128((__m128i *)eth_hdr); + addr0 = _mm_shuffle_epi8(addr0, shfl_msk); + _mm_storeu_si128((__m128i *)eth_hdr[0], addr0); - mbuf_field_set(mb, ol_flags); + mbuf_field_set(mb[0], ol_flags); } }