From patchwork Tue Jan 14 07:49:50 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Slava Ovsiienko X-Patchwork-Id: 64619 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 9D61DA04FD; Tue, 14 Jan 2020 08:50:19 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 9956E1C2A7; Tue, 14 Jan 2020 08:50:07 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 9469C1C29B for ; Tue, 14 Jan 2020 08:50:01 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from viacheslavo@mellanox.com) with ESMTPS (AES256-SHA encrypted); 14 Jan 2020 09:49:57 +0200 Received: from pegasus11.mtr.labs.mlnx (pegasus11.mtr.labs.mlnx [10.210.16.104]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 00E7nvlc008557; Tue, 14 Jan 2020 09:49:57 +0200 Received: from pegasus11.mtr.labs.mlnx (localhost [127.0.0.1]) by pegasus11.mtr.labs.mlnx (8.14.7/8.14.7) with ESMTP id 00E7nvH7009289; Tue, 14 Jan 2020 07:49:57 GMT Received: (from viacheslavo@localhost) by pegasus11.mtr.labs.mlnx (8.14.7/8.14.7/Submit) id 00E7nvD5009288; Tue, 14 Jan 2020 07:49:57 GMT X-Authentication-Warning: pegasus11.mtr.labs.mlnx: viacheslavo set sender to viacheslavo@mellanox.com using -f From: Viacheslav Ovsiienko To: dev@dpdk.org Cc: matan@mellanox.com, rasland@mellanox.com, orika@mellanox.com, shahafs@mellanox.com, olivier.matz@6wind.com, stephen@networkplumber.org Date: Tue, 14 Jan 2020 07:49:50 +0000 Message-Id: <1578988193-9015-2-git-send-email-viacheslavo@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1578988193-9015-1-git-send-email-viacheslavo@mellanox.com> References: <20191118094938.192850-1-shahafs@mellanox.com> <1578988193-9015-1-git-send-email-viacheslavo@mellanox.com> Subject: [dpdk-dev] [PATCH v2 1/4] mbuf: detach mbuf with pinned external buffer X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Update detach routine to check the mbuf pool type. Signed-off-by: Shahaf Shuler Signed-off-by: Viacheslav Ovsiienko --- lib/librte_mbuf/rte_mbuf.h | 65 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h index 219b110..46ae76c 100644 --- a/lib/librte_mbuf/rte_mbuf.h +++ b/lib/librte_mbuf/rte_mbuf.h @@ -32,6 +32,7 @@ */ #include +#include #include #include #include @@ -306,6 +307,46 @@ struct rte_pktmbuf_pool_private { uint32_t flags; /**< reserved for future use. */ }; +/** + * When set pktmbuf mempool will hold only mbufs with pinned external + * buffer. The external buffer will be attached on the mbuf at the + * memory pool creation and will never be detached by the mbuf free calls. + * mbuf should not contain any room for data after the mbuf structure. + */ +#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0) + +/** + * Returns TRUE if given mbuf has an pinned external buffer, or FALSE + * otherwise. The pinned external buffer is allocated at pool creation + * time and should not be freed. + * + * External buffer is a user-provided anonymous buffer. + */ +#ifdef ALLOW_EXPERIMENTAL_API +#define RTE_MBUF_HAS_PINNED_EXTBUF(mb) rte_mbuf_has_pinned_extbuf(mb) +#else +#define RTE_MBUF_HAS_PINNED_EXTBUF(mb) false +#endif + +__rte_experimental +static inline uint32_t +rte_mbuf_has_pinned_extbuf(const struct rte_mbuf *m) +{ + if (RTE_MBUF_HAS_EXTBUF(m)) { + /* + * The mbuf has the external attached buffer, + * we should check the type of the memory pool where + * the mbuf was allocated from. + */ + struct rte_pktmbuf_pool_private *priv = + (struct rte_pktmbuf_pool_private *) + rte_mempool_get_priv(m->pool); + + return priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF; + } + return 0; +} + #ifdef RTE_LIBRTE_MBUF_DEBUG /** check mbuf type in debug mode */ @@ -571,7 +612,8 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m) { - RTE_ASSERT(RTE_MBUF_DIRECT(m)); + RTE_ASSERT(!RTE_MBUF_CLONED(m) && + (!RTE_MBUF_HAS_EXTBUF(m) || RTE_MBUF_HAS_PINNED_EXTBUF(m))); RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); RTE_ASSERT(m->next == NULL); RTE_ASSERT(m->nb_segs == 1); @@ -1141,11 +1183,26 @@ static inline void rte_pktmbuf_detach(struct rte_mbuf *m) uint32_t mbuf_size, buf_len; uint16_t priv_size; - if (RTE_MBUF_HAS_EXTBUF(m)) + if (RTE_MBUF_HAS_EXTBUF(m)) { + /* + * The mbuf has the external attached buffed, + * we should check the type of the memory pool where + * the mbuf was allocated from to detect the pinned + * external buffer. + */ + struct rte_pktmbuf_pool_private *priv = + (struct rte_pktmbuf_pool_private *) + rte_mempool_get_priv(mp); + + if (priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) { + RTE_ASSERT(m->shinfo == NULL); + m->ol_flags = EXT_ATTACHED_MBUF; + return; + } __rte_pktmbuf_free_extbuf(m); - else + } else { __rte_pktmbuf_free_direct(m); - + } priv_size = rte_pktmbuf_priv_size(mp); mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size); buf_len = rte_pktmbuf_data_room_size(mp); From patchwork Tue Jan 14 07:49:52 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Slava Ovsiienko X-Patchwork-Id: 64620 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id D47C7A04FD; Tue, 14 Jan 2020 08:50:25 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 807A71C2B5; Tue, 14 Jan 2020 08:50:09 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 8EE091C294 for ; Tue, 14 Jan 2020 08:50:01 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from viacheslavo@mellanox.com) with ESMTPS (AES256-SHA encrypted); 14 Jan 2020 09:49:59 +0200 Received: from pegasus11.mtr.labs.mlnx (pegasus11.mtr.labs.mlnx [10.210.16.104]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 00E7nxSw008567; Tue, 14 Jan 2020 09:49:59 +0200 Received: from pegasus11.mtr.labs.mlnx (localhost [127.0.0.1]) by pegasus11.mtr.labs.mlnx (8.14.7/8.14.7) with ESMTP id 00E7nxiK009294; Tue, 14 Jan 2020 07:49:59 GMT Received: (from viacheslavo@localhost) by pegasus11.mtr.labs.mlnx (8.14.7/8.14.7/Submit) id 00E7nxDf009293; Tue, 14 Jan 2020 07:49:59 GMT X-Authentication-Warning: pegasus11.mtr.labs.mlnx: viacheslavo set sender to viacheslavo@mellanox.com using -f From: Viacheslav Ovsiienko To: dev@dpdk.org Cc: matan@mellanox.com, rasland@mellanox.com, orika@mellanox.com, shahafs@mellanox.com, olivier.matz@6wind.com, stephen@networkplumber.org Date: Tue, 14 Jan 2020 07:49:52 +0000 Message-Id: <1578988193-9015-4-git-send-email-viacheslavo@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1578988193-9015-1-git-send-email-viacheslavo@mellanox.com> References: <20191118094938.192850-1-shahafs@mellanox.com> <1578988193-9015-1-git-send-email-viacheslavo@mellanox.com> Subject: [dpdk-dev] [PATCH v2 3/4] app/testpmd: add mempool with external data buffers X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" The new mbuf pool type is added to testpmd. To engage the mbuf pool with externally attached data buffers the parameter "--mp-alloc=xbuf" should be specified in testpmd command line. The objective of this patch is just to test whether mbuf pool with externally attached data buffers works OK. The memory for data buffers is allocated from DPDK memory, so this is not "true" external memory from some physical device (this is supposed the most common use case for such kind of mbuf pool). The user should be aware that not all drivers support the mbuf with EXT_ATTACHED_BUF flags set in newly allocated mbuf (many PMDs just overwrite ol_flags field and flag value is getting lost). Signed-off-by: Viacheslav Ovsiienko --- app/test-pmd/config.c | 2 ++ app/test-pmd/flowgen.c | 3 +- app/test-pmd/parameters.c | 2 ++ app/test-pmd/testpmd.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++ app/test-pmd/testpmd.h | 4 ++- app/test-pmd/txonly.c | 3 +- 6 files changed, 92 insertions(+), 3 deletions(-) diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c index 9da1ffb..5c6fe18 100644 --- a/app/test-pmd/config.c +++ b/app/test-pmd/config.c @@ -2395,6 +2395,8 @@ struct igb_ring_desc_16_bytes { return "xmem"; case MP_ALLOC_XMEM_HUGE: return "xmemhuge"; + case MP_ALLOC_XBUF: + return "xbuf"; default: return "invalid"; } diff --git a/app/test-pmd/flowgen.c b/app/test-pmd/flowgen.c index 03b72aa..ae50cdc 100644 --- a/app/test-pmd/flowgen.c +++ b/app/test-pmd/flowgen.c @@ -199,7 +199,8 @@ sizeof(*ip_hdr)); pkt->nb_segs = 1; pkt->pkt_len = pkt_size; - pkt->ol_flags = ol_flags; + pkt->ol_flags &= EXT_ATTACHED_MBUF; + pkt->ol_flags |= ol_flags; pkt->vlan_tci = vlan_tci; pkt->vlan_tci_outer = vlan_tci_outer; pkt->l2_len = sizeof(struct rte_ether_hdr); diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c index 2e7a504..6340104 100644 --- a/app/test-pmd/parameters.c +++ b/app/test-pmd/parameters.c @@ -841,6 +841,8 @@ mp_alloc_type = MP_ALLOC_XMEM; else if (!strcmp(optarg, "xmemhuge")) mp_alloc_type = MP_ALLOC_XMEM_HUGE; + else if (!strcmp(optarg, "xbuf")) + mp_alloc_type = MP_ALLOC_XBUF; else rte_exit(EXIT_FAILURE, "mp-alloc %s invalid - must be: " diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c index 2eec8af..5f910ba 100644 --- a/app/test-pmd/testpmd.c +++ b/app/test-pmd/testpmd.c @@ -78,6 +78,7 @@ #endif #define EXTMEM_HEAP_NAME "extmem" +#define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M uint16_t verbose_level = 0; /**< Silent by default. */ int testpmd_logtype; /**< Log type for testpmd logs */ @@ -865,6 +866,66 @@ struct extmem_param { } } +static unsigned int +setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, + char *pool_name, struct rte_pktmbuf_extmem **ext_mem) +{ + struct rte_pktmbuf_extmem *xmem; + unsigned int ext_num, zone_num, elt_num; + uint16_t elt_size; + + elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); + elt_num = EXTBUF_ZONE_SIZE / elt_size; + zone_num = (nb_mbufs + elt_num - 1) / elt_num; + + xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); + if (xmem == NULL) { + TESTPMD_LOG(ERR, "Cannot allocate memory for " + "external buffer descriptors\n"); + *ext_mem = NULL; + return 0; + } + for (ext_num = 0; ext_num < zone_num; ext_num++) { + struct rte_pktmbuf_extmem *xseg = xmem + ext_num; + const struct rte_memzone *mz; + char mz_name[RTE_MEMZONE_NAMESIZE]; + int ret; + + ret = snprintf(mz_name, sizeof(mz_name), + RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); + if (ret < 0 || ret >= (int)sizeof(mz_name)) { + errno = ENAMETOOLONG; + ext_num = 0; + break; + } + mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, + socket_id, + RTE_MEMZONE_IOVA_CONTIG | + RTE_MEMZONE_1GB | + RTE_MEMZONE_SIZE_HINT_ONLY, + EXTBUF_ZONE_SIZE); + if (mz == NULL) { + /* + * The caller exits on external buffer creation + * error, so there is no need to free memzones. + */ + errno = ENOMEM; + ext_num = 0; + break; + } + xseg->buf_ptr = mz->addr; + xseg->buf_iova = mz->iova; + xseg->buf_len = EXTBUF_ZONE_SIZE; + xseg->elt_size = elt_size; + } + if (ext_num == 0 && xmem != NULL) { + free(xmem); + xmem = NULL; + } + *ext_mem = xmem; + return ext_num; +} + /* * Configuration initialisation done once at init time. */ @@ -933,6 +994,26 @@ struct extmem_param { heap_socket); break; } + case MP_ALLOC_XBUF: + { + struct rte_pktmbuf_extmem *ext_mem; + unsigned int ext_num; + + ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, + socket_id, pool_name, &ext_mem); + if (ext_num == 0) + rte_exit(EXIT_FAILURE, + "Can't create pinned data buffers\n"); + + TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", + rte_mbuf_best_mempool_ops()); + rte_mp = rte_pktmbuf_pool_create_extbuf + (pool_name, nb_mbuf, mb_mempool_cache, + 0, mbuf_seg_size, socket_id, + ext_mem, ext_num); + free(ext_mem); + break; + } default: { rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index 857a11f..a47f214 100644 --- a/app/test-pmd/testpmd.h +++ b/app/test-pmd/testpmd.h @@ -76,8 +76,10 @@ enum { /**< allocate mempool natively, but populate using anonymous memory */ MP_ALLOC_XMEM, /**< allocate and populate mempool using anonymous memory */ - MP_ALLOC_XMEM_HUGE + MP_ALLOC_XMEM_HUGE, /**< allocate and populate mempool using anonymous hugepage memory */ + MP_ALLOC_XBUF + /**< allocate mempool natively, use rte_pktmbuf_pool_create_extbuf */ }; #ifdef RTE_TEST_PMD_RECORD_BURST_STATS diff --git a/app/test-pmd/txonly.c b/app/test-pmd/txonly.c index 3caf281..871cf6c 100644 --- a/app/test-pmd/txonly.c +++ b/app/test-pmd/txonly.c @@ -170,7 +170,8 @@ rte_pktmbuf_reset_headroom(pkt); pkt->data_len = tx_pkt_seg_lengths[0]; - pkt->ol_flags = ol_flags; + pkt->ol_flags &= EXT_ATTACHED_MBUF; + pkt->ol_flags |= ol_flags; pkt->vlan_tci = vlan_tci; pkt->vlan_tci_outer = vlan_tci_outer; pkt->l2_len = sizeof(struct rte_ether_hdr); From patchwork Tue Jan 14 07:49:53 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Slava Ovsiienko X-Patchwork-Id: 64621 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6378CA04FD; Tue, 14 Jan 2020 08:50:36 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id D14621C2BF; Tue, 14 Jan 2020 08:50:10 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 97E9F1C29C for ; Tue, 14 Jan 2020 08:50:01 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from viacheslavo@mellanox.com) with ESMTPS (AES256-SHA encrypted); 14 Jan 2020 09:50:00 +0200 Received: from pegasus11.mtr.labs.mlnx (pegasus11.mtr.labs.mlnx [10.210.16.104]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 00E7o0X0008572; Tue, 14 Jan 2020 09:50:00 +0200 Received: from pegasus11.mtr.labs.mlnx (localhost [127.0.0.1]) by pegasus11.mtr.labs.mlnx (8.14.7/8.14.7) with ESMTP id 00E7o09M009296; Tue, 14 Jan 2020 07:50:00 GMT Received: (from viacheslavo@localhost) by pegasus11.mtr.labs.mlnx (8.14.7/8.14.7/Submit) id 00E7nx6O009295; Tue, 14 Jan 2020 07:50:00 GMT X-Authentication-Warning: pegasus11.mtr.labs.mlnx: viacheslavo set sender to viacheslavo@mellanox.com using -f From: Viacheslav Ovsiienko To: dev@dpdk.org Cc: matan@mellanox.com, rasland@mellanox.com, orika@mellanox.com, shahafs@mellanox.com, olivier.matz@6wind.com, stephen@networkplumber.org Date: Tue, 14 Jan 2020 07:49:53 +0000 Message-Id: <1578988193-9015-5-git-send-email-viacheslavo@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1578988193-9015-1-git-send-email-viacheslavo@mellanox.com> References: <20191118094938.192850-1-shahafs@mellanox.com> <1578988193-9015-1-git-send-email-viacheslavo@mellanox.com> Subject: [dpdk-dev] [PATCH v2 4/4] net/mlx5: allow use allocated mbuf with external buffer X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" In the Rx datapath the flags in the newly allocated mbufs are all explicitly cleared but the EXT_ATTACHED_MBUF must be preserved. It would allow to use mbuf pools with pre-attached external data buffers. The vectorized rx_burst routines are updated in order to inherit the EXT_ATTACHED_MBUF from mbuf pool private RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF flag. Signed-off-by: Viacheslav Ovsiienko Acked-by: Matan Azrad --- drivers/net/mlx5/mlx5_rxq.c | 7 ++++++- drivers/net/mlx5/mlx5_rxtx.c | 2 +- drivers/net/mlx5/mlx5_rxtx.h | 2 +- drivers/net/mlx5/mlx5_rxtx_vec.h | 14 ++++---------- drivers/net/mlx5/mlx5_rxtx_vec_altivec.h | 5 ++--- drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 29 +++++++++++++++-------------- drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 2 +- 7 files changed, 30 insertions(+), 31 deletions(-) diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index ca25e32..c87ce15 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -225,6 +225,9 @@ if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) { struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; + struct rte_pktmbuf_pool_private *priv = + (struct rte_pktmbuf_pool_private *) + rte_mempool_get_priv(rxq_ctrl->rxq.mp); int j; /* Initialize default rearm_data for vPMD. */ @@ -232,13 +235,15 @@ rte_mbuf_refcnt_set(mbuf_init, 1); mbuf_init->nb_segs = 1; mbuf_init->port = rxq->port_id; + if (priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) + mbuf_init->ol_flags = EXT_ATTACHED_MBUF; /* * prevent compiler reordering: * rearm_data covers previous fields. */ rte_compiler_barrier(); rxq->mbuf_initializer = - *(uint64_t *)&mbuf_init->rearm_data; + *(rte_xmm_t *)&mbuf_init->rearm_data; /* Padding with a fake mbuf for vectorized Rx. */ for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j) (*rxq->elts)[elts_n + j] = &rxq->fake_mbuf; diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index b11c5eb..fdc7529 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -1337,7 +1337,7 @@ enum mlx5_txcmp_code { } pkt = seg; assert(len >= (rxq->crc_present << 2)); - pkt->ol_flags = 0; + pkt->ol_flags &= EXT_ATTACHED_MBUF; /* If compressed, take hash result from mini-CQE. */ rss_hash_res = rte_be_to_cpu_32(mcqe == NULL ? cqe->rx_hash_res : diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index e362b4a..24fa038 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -144,7 +144,7 @@ struct mlx5_rxq_data { struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish. */ uint16_t idx; /* Queue index. */ struct mlx5_rxq_stats stats; - uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */ + rte_xmm_t mbuf_initializer; /* Default rearm/flags for vectorized Rx. */ struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */ void *cq_uar; /* CQ user access region. */ uint32_t cqn; /* CQ number. */ diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h index 85e0bd5..d8c07f2 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec.h @@ -97,18 +97,12 @@ void *buf_addr; /* - * Load the virtual address for Rx WQE. non-x86 processors - * (mostly RISC such as ARM and Power) are more vulnerable to - * load stall. For x86, reducing the number of instructions - * seems to matter most. + * In order to support the mbufs with external attached + * data buffer we should use the buf_addr pointer instead of + * rte_mbuf_buf_addr(). It touches the mbuf itself and may + * impact the performance. */ -#ifdef RTE_ARCH_X86_64 buf_addr = elts[i]->buf_addr; - assert(buf_addr == rte_mbuf_buf_addr(elts[i], rxq->mp)); -#else - buf_addr = rte_mbuf_buf_addr(elts[i], rxq->mp); - assert(buf_addr == elts[i]->buf_addr); -#endif wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr + RTE_PKTMBUF_HEADROOM); /* If there's only one MR, no need to replace LKey in WQE. */ diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h index 8e79883..9e5c6ee 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h @@ -344,9 +344,8 @@ PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED}; const vector unsigned char mbuf_init = - (vector unsigned char)(vector unsigned long){ - *(__attribute__((__aligned__(8))) unsigned long *) - &rxq->mbuf_initializer, 0LL}; + (vector unsigned char)vec_vsx_ld + (0, (vector unsigned char *)&rxq->mbuf_initializer); const vector unsigned short rearm_sel_mask = (vector unsigned short){0, 0, 0, 0, 0xffff, 0xffff, 0, 0}; vector unsigned char rearm0, rearm1, rearm2, rearm3; diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h index 86785c7..332e9ac 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h @@ -264,8 +264,8 @@ const uint32x4_t cv_mask = vdupq_n_u32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED); - const uint64x1_t mbuf_init = vld1_u64(&rxq->mbuf_initializer); - const uint64x1_t r32_mask = vcreate_u64(0xffffffff); + const uint64x2_t mbuf_init = vld1q_u64 + ((const uint64_t *)&rxq->mbuf_initializer); uint64x2_t rearm0, rearm1, rearm2, rearm3; uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3; @@ -326,18 +326,19 @@ /* Merge to ol_flags. */ ol_flags = vorrq_u32(ol_flags, cv_flags); /* Merge mbuf_init and ol_flags, and store. */ - rearm0 = vcombine_u64(mbuf_init, - vshr_n_u64(vget_high_u64(vreinterpretq_u64_u32( - ol_flags)), 32)); - rearm1 = vcombine_u64(mbuf_init, - vand_u64(vget_high_u64(vreinterpretq_u64_u32( - ol_flags)), r32_mask)); - rearm2 = vcombine_u64(mbuf_init, - vshr_n_u64(vget_low_u64(vreinterpretq_u64_u32( - ol_flags)), 32)); - rearm3 = vcombine_u64(mbuf_init, - vand_u64(vget_low_u64(vreinterpretq_u64_u32( - ol_flags)), r32_mask)); + rearm0 = vreinterpretq_u64_u32(vsetq_lane_u32 + (vgetq_lane_u32(ol_flags, 3), + vreinterpretq_u32_u64(mbuf_init), 2)); + rearm1 = vreinterpretq_u64_u32(vsetq_lane_u32 + (vgetq_lane_u32(ol_flags, 2), + vreinterpretq_u32_u64(mbuf_init), 2)); + rearm2 = vreinterpretq_u64_u32(vsetq_lane_u32 + (vgetq_lane_u32(ol_flags, 1), + vreinterpretq_u32_u64(mbuf_init), 2)); + rearm3 = vreinterpretq_u64_u32(vsetq_lane_u32 + (vgetq_lane_u32(ol_flags, 0), + vreinterpretq_u32_u64(mbuf_init), 2)); + vst1q_u64((void *)&pkts[0]->rearm_data, rearm0); vst1q_u64((void *)&pkts[1]->rearm_data, rearm1); vst1q_u64((void *)&pkts[2]->rearm_data, rearm2); diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h index 35b7761..07d40d5 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h @@ -259,7 +259,7 @@ PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED); const __m128i mbuf_init = - _mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer); + _mm_load_si128((__m128i *)&rxq->mbuf_initializer); __m128i rearm0, rearm1, rearm2, rearm3; uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;