From patchwork Thu Sep 15 06:58:24 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Joyce Kong X-Patchwork-Id: 116323 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id BBE96A00C5; Thu, 15 Sep 2022 08:58:45 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 671D34281A; Thu, 15 Sep 2022 08:58:41 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id A53B440223 for ; Thu, 15 Sep 2022 08:58:39 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id A027D1692; Wed, 14 Sep 2022 23:58:45 -0700 (PDT) Received: from net-arm-n1sdp-02.shanghai.arm.com (net-arm-n1sdp-02.shanghai.arm.com [10.169.210.140]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id ACC5E3F73B; Wed, 14 Sep 2022 23:59:05 -0700 (PDT) From: Joyce Kong To: jgrajcia@cisco.com, stephen@networkplumber.org Cc: dev@dpdk.org, nd@arm.com, Joyce Kong Subject: [PATCH v4 1/2] net/memif: add a Rx fast path Date: Thu, 15 Sep 2022 14:58:24 +0800 Message-Id: <20220915065825.26169-2-joyce.kong@arm.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20220915065825.26169-1-joyce.kong@arm.com> References: <20220412093243.3670187-1-joyce.kong@arm.com> <20220915065825.26169-1-joyce.kong@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org For memif non-zero-copy mode, there is a branch to compare the mbuf and memif buffer size during memory copying. Mbuf and memif buffer size is defined at compile time. If memif buf size <= mbuf size, add a fast Rx memory copy path by removing this branch and mbuf bulk alloc. The removal of the branch and bulk alloc lead to considerable performance uplift. Test with 1p1q on N1SDP AArch64 server, -------------------------------------------- buf size | memif <= mbuf | memif > mbuf | -------------------------------------------- non-zc gain | 26.85% | -0.37% | -------------------------------------------- zc gain | 8.57% | 3.04% | -------------------------------------------- Test with 1p1q on Cascade Lake Xeon X86 server, -------------------------------------------- buf size | memif <= mbuf | memif > mbuf | -------------------------------------------- non-zc gain | 17.54% | -0.42% | -------------------------------------------- zc gain | 10.67% | 0.26% | -------------------------------------------- Signed-off-by: Joyce Kong Reviewed-by: Ruifeng Wang Acked-by: Morten Brørup --- drivers/net/memif/rte_eth_memif.c | 146 +++++++++++++++++++++--------- drivers/net/memif/rte_eth_memif.h | 2 + 2 files changed, 107 insertions(+), 41 deletions(-) diff --git a/drivers/net/memif/rte_eth_memif.c b/drivers/net/memif/rte_eth_memif.c index dd951b8296..762293f636 100644 --- a/drivers/net/memif/rte_eth_memif.c +++ b/drivers/net/memif/rte_eth_memif.c @@ -297,7 +297,7 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) rte_eth_devices[mq->in_port].process_private; memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq); uint16_t cur_slot, last_slot, n_slots, ring_size, mask, s0; - uint16_t n_rx_pkts = 0; + uint16_t pkts, rx_pkts, n_rx_pkts = 0; uint16_t mbuf_size = rte_pktmbuf_data_room_size(mq->mempool) - RTE_PKTMBUF_HEADROOM; uint16_t src_len, src_off, dst_len, dst_off, cp_len; @@ -342,66 +342,131 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) goto refill; n_slots = last_slot - cur_slot; - while (n_slots && n_rx_pkts < nb_pkts) { - mbuf_head = rte_pktmbuf_alloc(mq->mempool); - if (unlikely(mbuf_head == NULL)) + if (likely(mbuf_size >= pmd->cfg.pkt_buffer_size)) { + struct rte_mbuf *mbufs[MAX_PKT_BURST]; +next_bulk: + ret = rte_pktmbuf_alloc_bulk(mq->mempool, mbufs, MAX_PKT_BURST); + if (unlikely(ret < 0)) goto no_free_bufs; - mbuf = mbuf_head; - mbuf->port = mq->in_port; - dst_off = 0; -next_slot: - s0 = cur_slot & mask; - d0 = &ring->desc[s0]; + rx_pkts = 0; + pkts = nb_pkts < MAX_PKT_BURST ? nb_pkts : MAX_PKT_BURST; + while (n_slots && rx_pkts < pkts) { + mbuf_head = mbufs[n_rx_pkts]; + mbuf = mbuf_head; - src_len = d0->length; - src_off = 0; +next_slot1: + mbuf->port = mq->in_port; + s0 = cur_slot & mask; + d0 = &ring->desc[s0]; - do { - dst_len = mbuf_size - dst_off; - if (dst_len == 0) { - dst_off = 0; - dst_len = mbuf_size; + cp_len = d0->length; + + rte_pktmbuf_data_len(mbuf) = cp_len; + rte_pktmbuf_pkt_len(mbuf) = cp_len; + if (mbuf != mbuf_head) + rte_pktmbuf_pkt_len(mbuf_head) += cp_len; + + rte_memcpy(rte_pktmbuf_mtod(mbuf, void *), + (uint8_t *)memif_get_buffer(proc_private, d0), cp_len); - /* store pointer to tail */ + cur_slot++; + n_slots--; + + if (d0->flags & MEMIF_DESC_FLAG_NEXT) { mbuf_tail = mbuf; mbuf = rte_pktmbuf_alloc(mq->mempool); - if (unlikely(mbuf == NULL)) + if (unlikely(mbuf == NULL)) { + rte_pktmbuf_free_bulk(mbufs + rx_pkts, + MAX_PKT_BURST - rx_pkts); goto no_free_bufs; - mbuf->port = mq->in_port; + } ret = memif_pktmbuf_chain(mbuf_head, mbuf_tail, mbuf); if (unlikely(ret < 0)) { MIF_LOG(ERR, "number-of-segments-overflow"); rte_pktmbuf_free(mbuf); + rte_pktmbuf_free_bulk(mbufs + rx_pkts, + MAX_PKT_BURST - rx_pkts); goto no_free_bufs; } + goto next_slot1; } - cp_len = RTE_MIN(dst_len, src_len); - rte_pktmbuf_data_len(mbuf) += cp_len; - rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf); - if (mbuf != mbuf_head) - rte_pktmbuf_pkt_len(mbuf_head) += cp_len; + mq->n_bytes += rte_pktmbuf_pkt_len(mbuf_head); + *bufs++ = mbuf_head; + rx_pkts++; + n_rx_pkts++; + } - rte_memcpy(rte_pktmbuf_mtod_offset(mbuf, void *, - dst_off), - (uint8_t *)memif_get_buffer(proc_private, d0) + - src_off, cp_len); + if (rx_pkts < MAX_PKT_BURST) { + rte_pktmbuf_free_bulk(mbufs + rx_pkts, MAX_PKT_BURST - rx_pkts); + } else { + nb_pkts -= rx_pkts; + if (nb_pkts) + goto next_bulk; + } + } else { + while (n_slots && n_rx_pkts < nb_pkts) { + mbuf_head = rte_pktmbuf_alloc(mq->mempool); + if (unlikely(mbuf_head == NULL)) + goto no_free_bufs; + mbuf = mbuf_head; + mbuf->port = mq->in_port; + +next_slot2: + s0 = cur_slot & mask; + d0 = &ring->desc[s0]; - src_off += cp_len; - dst_off += cp_len; - src_len -= cp_len; - } while (src_len); + src_len = d0->length; + dst_off = 0; + src_off = 0; - cur_slot++; - n_slots--; + do { + dst_len = mbuf_size - dst_off; + if (dst_len == 0) { + dst_off = 0; + dst_len = mbuf_size; + + /* store pointer to tail */ + mbuf_tail = mbuf; + mbuf = rte_pktmbuf_alloc(mq->mempool); + if (unlikely(mbuf == NULL)) + goto no_free_bufs; + mbuf->port = mq->in_port; + ret = memif_pktmbuf_chain(mbuf_head, mbuf_tail, mbuf); + if (unlikely(ret < 0)) { + MIF_LOG(ERR, "number-of-segments-overflow"); + rte_pktmbuf_free(mbuf); + goto no_free_bufs; + } + } + cp_len = RTE_MIN(dst_len, src_len); - if (d0->flags & MEMIF_DESC_FLAG_NEXT) - goto next_slot; + rte_pktmbuf_data_len(mbuf) += cp_len; + rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf); + if (mbuf != mbuf_head) + rte_pktmbuf_pkt_len(mbuf_head) += cp_len; - mq->n_bytes += rte_pktmbuf_pkt_len(mbuf_head); - *bufs++ = mbuf_head; - n_rx_pkts++; + rte_memcpy(rte_pktmbuf_mtod_offset(mbuf, void *, + dst_off), + (uint8_t *)memif_get_buffer(proc_private, d0) + + src_off, cp_len); + + src_off += cp_len; + dst_off += cp_len; + src_len -= cp_len; + } while (src_len); + + cur_slot++; + n_slots--; + + if (d0->flags & MEMIF_DESC_FLAG_NEXT) + goto next_slot2; + + mq->n_bytes += rte_pktmbuf_pkt_len(mbuf_head); + *bufs++ = mbuf_head; + n_rx_pkts++; + } } no_free_bufs: @@ -694,7 +759,6 @@ eth_memif_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) return n_tx_pkts; } - static int memif_tx_one_zc(struct pmd_process_private *proc_private, struct memif_queue *mq, memif_ring_t *ring, struct rte_mbuf *mbuf, const uint16_t mask, diff --git a/drivers/net/memif/rte_eth_memif.h b/drivers/net/memif/rte_eth_memif.h index 81e7dceae0..09928ecc86 100644 --- a/drivers/net/memif/rte_eth_memif.h +++ b/drivers/net/memif/rte_eth_memif.h @@ -25,6 +25,8 @@ #define ETH_MEMIF_DISC_STRING_SIZE 96 #define ETH_MEMIF_SECRET_SIZE 24 +#define MAX_PKT_BURST 32 + extern int memif_logtype; #define MIF_LOG(level, fmt, args...) \ From patchwork Thu Sep 15 06:58:25 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Joyce Kong X-Patchwork-Id: 116324 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A1A95A00C5; Thu, 15 Sep 2022 08:58:50 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3DE94427F7; Thu, 15 Sep 2022 08:58:43 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id F21E14281E for ; Thu, 15 Sep 2022 08:58:41 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id EC0B31CE0; Wed, 14 Sep 2022 23:58:47 -0700 (PDT) Received: from net-arm-n1sdp-02.shanghai.arm.com (net-arm-n1sdp-02.shanghai.arm.com [10.169.210.140]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 04CB83F73B; Wed, 14 Sep 2022 23:59:07 -0700 (PDT) From: Joyce Kong To: jgrajcia@cisco.com, stephen@networkplumber.org Cc: dev@dpdk.org, nd@arm.com, Joyce Kong Subject: [PATCH v4 2/2] net/memif: add a Tx fast path Date: Thu, 15 Sep 2022 14:58:25 +0800 Message-Id: <20220915065825.26169-3-joyce.kong@arm.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20220915065825.26169-1-joyce.kong@arm.com> References: <20220412093243.3670187-1-joyce.kong@arm.com> <20220915065825.26169-1-joyce.kong@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org For memif non-zero-copy mode, there is a branch to compare the mbuf and memif buffer size during memory copying. If all mbufs come from the same mempool, and memif buf size >= mbuf size, add a fast Tx memory copy path without the comparing branch and with mbuf bulk free, otherwise still run the original Tx path. The removal of the branch and bulk free lead to considerable performance uplift. Test with 1p1q on Ampere Altra AArch64 server, -------------------------------------------- buf size | memif >= mbuf | memif < mbuf | -------------------------------------------- non-zc gain | 10.82% | 0.04% | -------------------------------------------- zc gain | 8.86% | 3.18% | -------------------------------------------- Test with 1p1q on Cascade Lake Xeon X86server, -------------------------------------------- buf size | memif >= mbuf | memif < mbuf | -------------------------------------------- non-zc gain | 7.32% | -0.85% | -------------------------------------------- zc gain | 12.75% | -0.16% | -------------------------------------------- Signed-off-by: Joyce Kong Reviewed-by: Ruifeng Wang Acked-by: Morten Brørup --- drivers/net/memif/rte_eth_memif.c | 134 ++++++++++++++++++++---------- 1 file changed, 92 insertions(+), 42 deletions(-) diff --git a/drivers/net/memif/rte_eth_memif.c b/drivers/net/memif/rte_eth_memif.c index 762293f636..b5bed955ed 100644 --- a/drivers/net/memif/rte_eth_memif.c +++ b/drivers/net/memif/rte_eth_memif.c @@ -680,62 +680,112 @@ eth_memif_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) n_free = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE) - slot; } - while (n_tx_pkts < nb_pkts && n_free) { - mbuf_head = *bufs++; - nb_segs = mbuf_head->nb_segs; - mbuf = mbuf_head; + uint8_t i; + struct rte_mbuf **buf_tmp = bufs; + mbuf_head = *buf_tmp++; + struct rte_mempool *mp = mbuf_head->pool; + + for (i = 1; i < nb_pkts; i++) { + mbuf_head = *buf_tmp++; + if (mbuf_head->pool != mp) + break; + } + + uint16_t mbuf_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM; + if (i == nb_pkts && pmd->cfg.pkt_buffer_size >= mbuf_size) { + buf_tmp = bufs; + while (n_tx_pkts < nb_pkts && n_free) { + mbuf_head = *bufs++; + nb_segs = mbuf_head->nb_segs; + mbuf = mbuf_head; - saved_slot = slot; - d0 = &ring->desc[slot & mask]; - dst_off = 0; - dst_len = (type == MEMIF_RING_C2S) ? - pmd->run.pkt_buffer_size : d0->length; + saved_slot = slot; -next_in_chain: - src_off = 0; - src_len = rte_pktmbuf_data_len(mbuf); +next_in_chain1: + d0 = &ring->desc[slot & mask]; + cp_len = rte_pktmbuf_data_len(mbuf); - while (src_len) { - if (dst_len == 0) { + rte_memcpy((uint8_t *)memif_get_buffer(proc_private, d0), + rte_pktmbuf_mtod(mbuf, void *), cp_len); + + d0->length = cp_len; + mq->n_bytes += cp_len; + slot++; + n_free--; + + if (--nb_segs > 0) { if (n_free) { - slot++; - n_free--; d0->flags |= MEMIF_DESC_FLAG_NEXT; - d0 = &ring->desc[slot & mask]; - dst_off = 0; - dst_len = (type == MEMIF_RING_C2S) ? - pmd->run.pkt_buffer_size : d0->length; - d0->flags = 0; + mbuf = mbuf->next; + goto next_in_chain1; } else { slot = saved_slot; - goto no_free_slots; + goto free_mbufs; } } - cp_len = RTE_MIN(dst_len, src_len); - rte_memcpy((uint8_t *)memif_get_buffer(proc_private, - d0) + dst_off, - rte_pktmbuf_mtod_offset(mbuf, void *, src_off), - cp_len); + n_tx_pkts++; + } +free_mbufs: + rte_pktmbuf_free_bulk(buf_tmp, n_tx_pkts); + } else { + while (n_tx_pkts < nb_pkts && n_free) { + mbuf_head = *bufs++; + nb_segs = mbuf_head->nb_segs; + mbuf = mbuf_head; - mq->n_bytes += cp_len; - src_off += cp_len; - dst_off += cp_len; - src_len -= cp_len; - dst_len -= cp_len; + saved_slot = slot; + d0 = &ring->desc[slot & mask]; + dst_off = 0; + dst_len = (type == MEMIF_RING_C2S) ? + pmd->run.pkt_buffer_size : d0->length; - d0->length = dst_off; - } +next_in_chain2: + src_off = 0; + src_len = rte_pktmbuf_data_len(mbuf); - if (--nb_segs > 0) { - mbuf = mbuf->next; - goto next_in_chain; - } + while (src_len) { + if (dst_len == 0) { + if (n_free) { + slot++; + n_free--; + d0->flags |= MEMIF_DESC_FLAG_NEXT; + d0 = &ring->desc[slot & mask]; + dst_off = 0; + dst_len = (type == MEMIF_RING_C2S) ? + pmd->run.pkt_buffer_size : d0->length; + d0->flags = 0; + } else { + slot = saved_slot; + goto no_free_slots; + } + } + cp_len = RTE_MIN(dst_len, src_len); - n_tx_pkts++; - slot++; - n_free--; - rte_pktmbuf_free(mbuf_head); + rte_memcpy((uint8_t *)memif_get_buffer(proc_private, + d0) + dst_off, + rte_pktmbuf_mtod_offset(mbuf, void *, src_off), + cp_len); + + mq->n_bytes += cp_len; + src_off += cp_len; + dst_off += cp_len; + src_len -= cp_len; + dst_len -= cp_len; + + d0->length = dst_off; + } + + if (--nb_segs > 0) { + mbuf = mbuf->next; + goto next_in_chain2; + } + + n_tx_pkts++; + slot++; + n_free--; + rte_pktmbuf_free(mbuf_head); + } } no_free_slots: