From patchwork Wed Sep 9 15:52:54 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77065 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E979CA04B5; Wed, 9 Sep 2020 17:53:24 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 62ED51C0CE; Wed, 9 Sep 2020 17:53:14 +0200 (CEST) Received: from mail-pj1-f67.google.com (mail-pj1-f67.google.com [209.85.216.67]) by dpdk.org (Postfix) with ESMTP id 7D0631C0CD for ; Wed, 9 Sep 2020 17:53:13 +0200 (CEST) Received: by mail-pj1-f67.google.com with SMTP id kk9so1560555pjb.2 for ; Wed, 09 Sep 2020 08:53:13 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=ARxLoz6AxiMLK1VeQ63z6GCuJpZN1cUAzC3kNwlnzyA=; b=N3WwPHXrxEl+OYjIjlFTJpqH9IEsJy+IuRxyu4sQto4J9uev6Atm6rtKtrWZ05O9/p WwsvzR349rPCZhcQRvwxoLpfpQeJ8yu2Hibm6RQ1sezE/BF+2cBnICQFZWR0CWbrm/9I qSN1sN+WzOBhNw4ZiiSRuSmDMHe0LpXu1jrq4= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=ARxLoz6AxiMLK1VeQ63z6GCuJpZN1cUAzC3kNwlnzyA=; b=aoxl7TrqAmIvDMRaMUAD/DGX9HA8E6TmkIFA4biEyq7do3RMWfP8LzdgJ+Q6Mcv8mR A4kWtSW6SOOE52ekFzVA4Fgi2ztEw0fwU7VXK3uE9ATEcOwECVykC3WyFcjHWR8rpnnD FV9BttcnSU2RcdMLIqFhEqTgMaBWh88xZYx397GEVgHK1vB7JDry2k1Eh0v8Lmn29KQj PraTsNNtV54qU39OXJjcPXp6Z4DiDW5hz9wqXYH6K/QW1j+HEbnr+FFIShi+/Pz2ds3g R8fYZln8CTq8Ds7CWDpMob69tbBiY1jd9Andwyj4s3ZF8FrCForioHWBVIbEkxyfgv3U rC6A== X-Gm-Message-State: AOAM533VnKVTq7OHzLz0ZknIoxRGR3/4nFmbINgAlB++N80dQVk+CnXn YBIKo4r+BZPbwyNeTZIdt3Wcj0YSu6iO9w== X-Google-Smtp-Source: ABdhPJxiFnUOqRwwIth7HqcwETulkMl/rCay1AWrzxNemNrS7P90ub+wnFmRtOyxQbQaa+Kb5I378w== X-Received: by 2002:a17:90a:414d:: with SMTP id m13mr1287679pjg.163.1599666792674; Wed, 09 Sep 2020 08:53:12 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.11 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:11 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur , Ruifeng Wang Cc: dev@dpdk.org, stable@dpdk.org Date: Wed, 9 Sep 2020 11:52:54 -0400 Message-Id: <20200909155302.28656-2-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 01/12] net/bnxt: fix burst mode get for Arm X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Transmit and receive burst mode get operations incorrectly return "Vector SSE" on ARM64 platforms, change to return "Vector Neon" instead. Fixes: 3983583414 ("net/bnxt: support NEON") Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson Cc: stable@dpdk.org --- drivers/net/bnxt/bnxt_ethdev.c | 60 +++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 75d055be00..7a77922c0c 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -2615,46 +2615,62 @@ bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.tx_deferred_start = txq->tx_deferred_start; } +static const struct { + eth_rx_burst_t pkt_burst; + const char *info; +} bnxt_rx_burst_info[] = { + {bnxt_recv_pkts, "Scalar"}, +#if defined(RTE_ARCH_X86) + {bnxt_recv_pkts_vec, "Vector SSE"}, +#elif defined(RTE_ARCH_ARM64) + {bnxt_recv_pkts_vec, "Vector Neon"}, +#endif +}; + static int bnxt_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode) { eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; + size_t i; - if (pkt_burst == bnxt_recv_pkts) { - snprintf(mode->info, sizeof(mode->info), "%s", - "Scalar"); - return 0; - } -#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) - if (pkt_burst == bnxt_recv_pkts_vec) { - snprintf(mode->info, sizeof(mode->info), "%s", - "Vector SSE"); - return 0; + for (i = 0; i < RTE_DIM(bnxt_rx_burst_info); i++) { + if (pkt_burst == bnxt_rx_burst_info[i].pkt_burst) { + snprintf(mode->info, sizeof(mode->info), "%s", + bnxt_rx_burst_info[i].info); + return 0; + } } -#endif return -EINVAL; } +static const struct { + eth_tx_burst_t pkt_burst; + const char *info; +} bnxt_tx_burst_info[] = { + {bnxt_xmit_pkts, "Scalar"}, +#if defined(RTE_ARCH_X86) + {bnxt_xmit_pkts_vec, "Vector SSE"}, +#elif defined(RTE_ARCH_ARM64) + {bnxt_xmit_pkts_vec, "Vector Neon"}, +#endif +}; + static int bnxt_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, struct rte_eth_burst_mode *mode) { eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; + size_t i; - if (pkt_burst == bnxt_xmit_pkts) { - snprintf(mode->info, sizeof(mode->info), "%s", - "Scalar"); - return 0; - } -#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) - if (pkt_burst == bnxt_xmit_pkts_vec) { - snprintf(mode->info, sizeof(mode->info), "%s", - "Vector SSE"); - return 0; + for (i = 0; i < RTE_DIM(bnxt_tx_burst_info); i++) { + if (pkt_burst == bnxt_tx_burst_info[i].pkt_burst) { + snprintf(mode->info, sizeof(mode->info), "%s", + bnxt_tx_burst_info[i].info); + return 0; + } } -#endif return -EINVAL; } From patchwork Wed Sep 9 15:52:55 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77066 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1F264A04B5; Wed, 9 Sep 2020 17:53:36 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id C03C11C0DA; Wed, 9 Sep 2020 17:53:17 +0200 (CEST) Received: from mail-pg1-f195.google.com (mail-pg1-f195.google.com [209.85.215.195]) by dpdk.org (Postfix) with ESMTP id 959111C0D8 for ; Wed, 9 Sep 2020 17:53:16 +0200 (CEST) Received: by mail-pg1-f195.google.com with SMTP id l191so2370804pgd.5 for ; Wed, 09 Sep 2020 08:53:16 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=YRBN/CdhQzqL2wKVUJ6XvcS3QoNs8bM9dtvQIQt8uEs=; b=YArTXszxofgQ8Uj+QVDlDICDdlqxVWF23fuUYKI0CxJhdGQjX0FdjQF9DfJVez7R43 0hwW12dz7wCifShPY+sXHc+LsBUqknKqSm8r4huRvmquT2llNP+BwuJZ7Q+DGMsmeMZs yOIb5soPDIJsBYm7I9yff6h+vb98dF+VBWSkw= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=YRBN/CdhQzqL2wKVUJ6XvcS3QoNs8bM9dtvQIQt8uEs=; b=fGumya7/vFXJ4085ir9lA7teGbQeZ83/+st9CLffZ0esf1lIy5UdH+CPWaBMZD5JkE dIjnewAJ/YNVS6iV5GNCUp6aI/CG5Nbtdqmxd2bnX0W1ePrRE3DSKTc+qmMHEZoC8SFF pNk3CzTKW53v0Bcq4aWrxS/OsYaSZ803HrngO/e9aU3zXFQEgDXQbZ7FHrI9cx8BRHPK iNmyZ0tLBHxxTxGD4xiJmPyBUp22cB43mncr8aIKq0lR9KdD/aOn8aHGtDc994qF+H9h Gni3tYBOjXfJGcjDEJAoxLMFWZ0RLfA1yunbTap8r7ZyaA+njzWnKLhTGRb1YFa/hptW bzqQ== X-Gm-Message-State: AOAM533OgklntK5BAGdUz+LUkg/bT+VVVkfsaeI8wC0Dlt6CM8hK6xtB XHt48Wua8oQD6GLk9WwIwi4gMQ== X-Google-Smtp-Source: ABdhPJwetxerF+1pWq8zyCzYYUM6+AdtzyNACcUJ3YQOH0jME2j3gfFpYBRi7jKGUcaH4Ju4rQd6LA== X-Received: by 2002:a05:6a00:8c5:b029:13e:ce2c:88bd with SMTP id s5-20020a056a0008c5b029013ece2c88bdmr1389079pfu.0.1599666795702; Wed, 09 Sep 2020 08:53:15 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.14 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:15 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org, stable@dpdk.org Date: Wed, 9 Sep 2020 11:52:55 -0400 Message-Id: <20200909155302.28656-3-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 02/12] net/bnxt: fix rxq/txq get information X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Return correct values for Rx/Tx offloads and for rx_drop_en. Fixes: 2fc201884be8 ("net/bnxt: support rxq/txq get information") Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson Cc: stable@dpdk.org --- drivers/net/bnxt/bnxt_ethdev.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 7a77922c0c..5585f872d0 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -2588,8 +2588,9 @@ bnxt_rxq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->nb_desc = rxq->nb_rx_desc; qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; - qinfo->conf.rx_drop_en = 0; + qinfo->conf.rx_drop_en = 1; qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; + qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; } static void @@ -2613,6 +2614,7 @@ bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.tx_free_thresh = txq->tx_free_thresh; qinfo->conf.tx_rs_thresh = 0; qinfo->conf.tx_deferred_start = txq->tx_deferred_start; + qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; } static const struct { From patchwork Wed Sep 9 15:52:56 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77067 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id A4B71A04B5; Wed, 9 Sep 2020 17:53:46 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 06A891C112; Wed, 9 Sep 2020 17:53:21 +0200 (CEST) Received: from mail-pf1-f194.google.com (mail-pf1-f194.google.com [209.85.210.194]) by dpdk.org (Postfix) with ESMTP id 92AEF1C10E for ; Wed, 9 Sep 2020 17:53:19 +0200 (CEST) Received: by mail-pf1-f194.google.com with SMTP id n14so2525422pff.6 for ; Wed, 09 Sep 2020 08:53:19 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=hD3h66OJlJ0Sb9lO78dYGSsWP+O2VlV5JJXboZ1OZkQ=; b=SLeSEh8ZUvhiLsrsIb9b5q5ffIuOIfV2kjQ48B3ZmGemTlI8yxBUMWuQmCVRl3d3KB 3MpvbodiU9R2E7VaPSy5jU1DstAZnmeKk5oZyt9/3E/gGUPgtCXgZiuYFueo7uRafAkh G3i/cBSyzzYzeFO1q6BMAVEAsaExBi2/2h3uQ= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=hD3h66OJlJ0Sb9lO78dYGSsWP+O2VlV5JJXboZ1OZkQ=; b=q54JYYI6biGuPSje3lDR7X5ycb3x6RQPQvBsPnbLY/HqxewrlcMGrI4lTOTPzJl2jn W5MNESOiPSs9GYgk5IAcUn8WqfYiO6g+1f5mwZKgXH13h4yrkbN8wrAaiUwyuKoQ39U3 58lakVwxCQ0RFghW8Pz1vebJIP4mdgc2KoYpY7qLyQH1+knmytXOVPhxsKiizFU79EQX 4Cz/WruPs04on/sLSIAHQTrERZYyxzU+5jIqgs6wmkIFNJdLGCJcBPXy6rmKyac9Ifz4 0QbD3XHhqdAuDDJLYUpygFMwImgL2vwqVb6CD3jfYcG9hm8ceYtFaOag5E33RCVcZxkJ fozg== X-Gm-Message-State: AOAM531xGVs246psThO2Ik6iCLm55hVI4LKaqmpR9Pmun/RpGgMPSZ4i ftJjKfxssn+8s43bwPOp/W2TBA== X-Google-Smtp-Source: ABdhPJxVO5Jsth63jREMjRkjx+oqbflt+VR2CGjEyqorohw7afCgiEuOeMhbWXUCnx+w2RrHTTtpcg== X-Received: by 2002:aa7:8646:: with SMTP id a6mr1379319pfo.54.1599666798531; Wed, 09 Sep 2020 08:53:18 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.17 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:18 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:52:56 -0400 Message-Id: <20200909155302.28656-4-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 03/12] net/bnxt: use appropriate type for Rx mbuf ring X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Change the type of the software receive mbuf ring from an array of structures containing an mbuf pointer to an array of pointers to struct rte_mbuf for consistency with how this ring is currently used by the vector mode receive function. Reviewed-by: Ajit Kumar Khaparde Reviewed-by: Somnath Kotur Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_ethdev.c | 6 ++-- drivers/net/bnxt/bnxt_reps.c | 21 +++++++------- drivers/net/bnxt/bnxt_ring.c | 4 +-- drivers/net/bnxt/bnxt_rxq.c | 14 ++++----- drivers/net/bnxt/bnxt_rxr.c | 41 ++++++++++++++------------- drivers/net/bnxt/bnxt_rxr.h | 8 ++---- drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 10 +++---- drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 10 +++---- 8 files changed, 55 insertions(+), 59 deletions(-) diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 5585f872d0..c57c5cc2af 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -2843,7 +2843,7 @@ bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset) struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue; struct bnxt_rx_ring_info *rxr; struct bnxt_cp_ring_info *cpr; - struct bnxt_sw_rx_bd *rx_buf; + struct rte_mbuf *rx_buf; struct rx_pkt_cmpl *rxcmp; uint32_t cons, cp_cons; int rc; @@ -2872,8 +2872,8 @@ bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset) if (CMPL_VALID(rxcmp, !cpr->valid)) return RTE_ETH_RX_DESC_DONE; } - rx_buf = &rxr->rx_buf_ring[cons]; - if (rx_buf->mbuf == NULL) + rx_buf = rxr->rx_buf_ring[cons]; + if (rx_buf == NULL) return RTE_ETH_RX_DESC_UNAVAIL; diff --git a/drivers/net/bnxt/bnxt_reps.c b/drivers/net/bnxt/bnxt_reps.c index a1b2c4bf97..bea9f3d1c1 100644 --- a/drivers/net/bnxt/bnxt_reps.c +++ b/drivers/net/bnxt/bnxt_reps.c @@ -35,7 +35,7 @@ static const struct eth_dev_ops bnxt_vf_rep_dev_ops = { uint16_t bnxt_vfr_recv(uint16_t port_id, uint16_t queue_id, struct rte_mbuf *mbuf) { - struct bnxt_sw_rx_bd *prod_rx_buf; + struct rte_mbuf **prod_rx_buf; struct bnxt_rx_ring_info *rep_rxr; struct bnxt_rx_queue *rep_rxq; struct rte_eth_dev *vfr_eth_dev; @@ -54,10 +54,9 @@ bnxt_vfr_recv(uint16_t port_id, uint16_t queue_id, struct rte_mbuf *mbuf) mask = rep_rxr->rx_ring_struct->ring_mask; /* Put this mbuf on the RxQ of the Representor */ - prod_rx_buf = - &rep_rxr->rx_buf_ring[rep_rxr->rx_prod++ & mask]; - if (!prod_rx_buf->mbuf) { - prod_rx_buf->mbuf = mbuf; + prod_rx_buf = &rep_rxr->rx_buf_ring[rep_rxr->rx_prod++ & mask]; + if (!*prod_rx_buf) { + *prod_rx_buf = mbuf; vfr_bp->rx_bytes[que] += mbuf->pkt_len; vfr_bp->rx_pkts[que]++; } else { @@ -75,7 +74,7 @@ bnxt_vf_rep_rx_burst(void *rx_queue, uint16_t nb_pkts) { struct bnxt_rx_queue *rxq = rx_queue; - struct bnxt_sw_rx_bd *cons_rx_buf; + struct rte_mbuf **cons_rx_buf; struct bnxt_rx_ring_info *rxr; uint16_t nb_rx_pkts = 0; uint16_t mask, i; @@ -87,11 +86,11 @@ bnxt_vf_rep_rx_burst(void *rx_queue, mask = rxr->rx_ring_struct->ring_mask; for (i = 0; i < nb_pkts; i++) { cons_rx_buf = &rxr->rx_buf_ring[rxr->rx_cons & mask]; - if (!cons_rx_buf->mbuf) + if (*cons_rx_buf == NULL) return nb_rx_pkts; - rx_pkts[nb_rx_pkts] = cons_rx_buf->mbuf; + rx_pkts[nb_rx_pkts] = *cons_rx_buf; rx_pkts[nb_rx_pkts]->port = rxq->port_id; - cons_rx_buf->mbuf = NULL; + *cons_rx_buf = NULL; nb_rx_pkts++; rxr->rx_cons++; } @@ -559,7 +558,7 @@ int bnxt_vf_rep_rx_queue_setup_op(struct rte_eth_dev *eth_dev, struct bnxt *parent_bp = rep_bp->parent_dev->data->dev_private; struct bnxt_rx_queue *parent_rxq; struct bnxt_rx_queue *rxq; - struct bnxt_sw_rx_bd *buf_ring; + struct rte_mbuf **buf_ring; int rc = 0; if (queue_idx >= BNXT_MAX_VF_REP_RINGS) { @@ -611,7 +610,7 @@ int bnxt_vf_rep_rx_queue_setup_op(struct rte_eth_dev *eth_dev, goto out; buf_ring = rte_zmalloc_socket("bnxt_rx_vfr_buf_ring", - sizeof(struct bnxt_sw_rx_bd) * + sizeof(struct rte_mbuf *) * rxq->rx_ring->rx_ring_struct->ring_size, RTE_CACHE_LINE_SIZE, socket_id); if (!buf_ring) { diff --git a/drivers/net/bnxt/bnxt_ring.c b/drivers/net/bnxt/bnxt_ring.c index 8f2296b293..f7f6ee8049 100644 --- a/drivers/net/bnxt/bnxt_ring.c +++ b/drivers/net/bnxt/bnxt_ring.c @@ -251,7 +251,7 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, rx_ring->vmem = (void **)((char *)mz->addr + rx_vmem_start); rx_ring_info->rx_buf_ring = - (struct bnxt_sw_rx_bd *)rx_ring->vmem; + (struct rte_mbuf **)rx_ring->vmem; } rx_ring = rx_ring_info->ag_ring_struct; @@ -269,7 +269,7 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, rx_ring->vmem = (void **)((char *)mz->addr + ag_vmem_start); rx_ring_info->ag_buf_ring = - (struct bnxt_sw_rx_bd *)rx_ring->vmem; + (struct rte_mbuf **)rx_ring->vmem; } rx_ring_info->ag_bitmap = diff --git a/drivers/net/bnxt/bnxt_rxq.c b/drivers/net/bnxt/bnxt_rxq.c index e42308a97f..db9aa1f3ed 100644 --- a/drivers/net/bnxt/bnxt_rxq.c +++ b/drivers/net/bnxt/bnxt_rxq.c @@ -197,7 +197,7 @@ int bnxt_mq_rx_configure(struct bnxt *bp) void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) { - struct bnxt_sw_rx_bd *sw_ring; + struct rte_mbuf **sw_ring; struct bnxt_tpa_info *tpa_info; uint16_t i; @@ -210,9 +210,9 @@ void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) if (sw_ring) { for (i = 0; i < rxq->rx_ring->rx_ring_struct->ring_size; i++) { - if (sw_ring[i].mbuf) { - rte_pktmbuf_free_seg(sw_ring[i].mbuf); - sw_ring[i].mbuf = NULL; + if (sw_ring[i]) { + rte_pktmbuf_free_seg(sw_ring[i]); + sw_ring[i] = NULL; } } } @@ -221,9 +221,9 @@ void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) if (sw_ring) { for (i = 0; i < rxq->rx_ring->ag_ring_struct->ring_size; i++) { - if (sw_ring[i].mbuf) { - rte_pktmbuf_free_seg(sw_ring[i].mbuf); - sw_ring[i].mbuf = NULL; + if (sw_ring[i]) { + rte_pktmbuf_free_seg(sw_ring[i]); + sw_ring[i] = NULL; } } } diff --git a/drivers/net/bnxt/bnxt_rxr.c b/drivers/net/bnxt/bnxt_rxr.c index b086898148..92102e3d57 100644 --- a/drivers/net/bnxt/bnxt_rxr.c +++ b/drivers/net/bnxt/bnxt_rxr.c @@ -42,7 +42,7 @@ static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, uint16_t prod) { struct rx_prod_pkt_bd *rxbd = &rxr->rx_desc_ring[prod]; - struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; + struct rte_mbuf **rx_buf = &rxr->rx_buf_ring[prod]; struct rte_mbuf *mbuf; mbuf = __bnxt_alloc_rx_data(rxq->mb_pool); @@ -51,7 +51,7 @@ static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, return -ENOMEM; } - rx_buf->mbuf = mbuf; + *rx_buf = mbuf; mbuf->data_off = RTE_PKTMBUF_HEADROOM; rxbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); @@ -64,7 +64,7 @@ static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq, uint16_t prod) { struct rx_prod_pkt_bd *rxbd = &rxr->ag_desc_ring[prod]; - struct bnxt_sw_rx_bd *rx_buf = &rxr->ag_buf_ring[prod]; + struct rte_mbuf **rx_buf = &rxr->ag_buf_ring[prod]; struct rte_mbuf *mbuf; if (rxbd == NULL) { @@ -83,7 +83,7 @@ static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq, return -ENOMEM; } - rx_buf->mbuf = mbuf; + *rx_buf = mbuf; mbuf->data_off = RTE_PKTMBUF_HEADROOM; rxbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); @@ -95,15 +95,15 @@ static inline void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, struct rte_mbuf *mbuf) { uint16_t prod = RING_NEXT(rxr->rx_ring_struct, rxr->rx_prod); - struct bnxt_sw_rx_bd *prod_rx_buf; + struct rte_mbuf **prod_rx_buf; struct rx_prod_pkt_bd *prod_bd; prod_rx_buf = &rxr->rx_buf_ring[prod]; - RTE_ASSERT(prod_rx_buf->mbuf == NULL); + RTE_ASSERT(*prod_rx_buf == NULL); RTE_ASSERT(mbuf != NULL); - prod_rx_buf->mbuf = mbuf; + *prod_rx_buf = mbuf; prod_bd = &rxr->rx_desc_ring[prod]; @@ -116,13 +116,14 @@ static inline struct rte_mbuf *bnxt_consume_rx_buf(struct bnxt_rx_ring_info *rxr, uint16_t cons) { - struct bnxt_sw_rx_bd *cons_rx_buf; + struct rte_mbuf **cons_rx_buf; struct rte_mbuf *mbuf; cons_rx_buf = &rxr->rx_buf_ring[cons]; - RTE_ASSERT(cons_rx_buf->mbuf != NULL); - mbuf = cons_rx_buf->mbuf; - cons_rx_buf->mbuf = NULL; + RTE_ASSERT(*cons_rx_buf != NULL); + mbuf = *cons_rx_buf; + *cons_rx_buf = NULL; + return mbuf; } @@ -226,7 +227,7 @@ static int bnxt_rx_pages(struct bnxt_rx_queue *rxq, bool is_thor_tpa = tpa_info && BNXT_CHIP_THOR(rxq->bp); for (i = 0; i < agg_buf; i++) { - struct bnxt_sw_rx_bd *ag_buf; + struct rte_mbuf **ag_buf; struct rte_mbuf *ag_mbuf; if (is_thor_tpa) { @@ -245,7 +246,7 @@ static int bnxt_rx_pages(struct bnxt_rx_queue *rxq, ag_cons = rxcmp->opaque; RTE_ASSERT(ag_cons <= rxr->ag_ring_struct->ring_mask); ag_buf = &rxr->ag_buf_ring[ag_cons]; - ag_mbuf = ag_buf->mbuf; + ag_mbuf = *ag_buf; RTE_ASSERT(ag_mbuf != NULL); ag_mbuf->data_len = rte_le_to_cpu_16(rxcmp->len); @@ -256,7 +257,7 @@ static int bnxt_rx_pages(struct bnxt_rx_queue *rxq, last->next = ag_mbuf; last = ag_mbuf; - ag_buf->mbuf = NULL; + *ag_buf = NULL; /* * As aggregation buffer consumed out of order in TPA module, @@ -866,10 +867,10 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, for (; cnt; i = RING_NEXT(rxr->rx_ring_struct, i), cnt--) { - struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; + struct rte_mbuf **rx_buf = &rxr->rx_buf_ring[i]; /* Buffer already allocated for this index. */ - if (rx_buf->mbuf != NULL) + if (*rx_buf != NULL) continue; /* This slot is empty. Alloc buffer for Rx */ @@ -960,7 +961,7 @@ int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) ring->ring_mask = ring->ring_size - 1; ring->bd = (void *)rxr->rx_desc_ring; ring->bd_dma = rxr->rx_desc_mapping; - ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd); + ring->vmem_size = ring->ring_size * sizeof(struct rte_mbuf *); ring->vmem = (void **)&rxr->rx_buf_ring; ring->fw_ring_id = INVALID_HW_RING_ID; @@ -998,7 +999,7 @@ int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) ring->ring_mask = ring->ring_size - 1; ring->bd = (void *)rxr->ag_desc_ring; ring->bd_dma = rxr->ag_desc_mapping; - ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd); + ring->vmem_size = ring->ring_size * sizeof(struct rte_mbuf *); ring->vmem = (void **)&rxr->ag_buf_ring; ring->fw_ring_id = INVALID_HW_RING_ID; @@ -1039,7 +1040,7 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) prod = rxr->rx_prod; for (i = 0; i < ring->ring_size; i++) { - if (unlikely(!rxr->rx_buf_ring[i].mbuf)) { + if (unlikely(!rxr->rx_buf_ring[i])) { if (bnxt_alloc_rx_data(rxq, rxr, prod) != 0) { PMD_DRV_LOG(WARNING, "init'ed rx ring %d with %d/%d mbufs only\n", @@ -1057,7 +1058,7 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) prod = rxr->ag_prod; for (i = 0; i < ring->ring_size; i++) { - if (unlikely(!rxr->ag_buf_ring[i].mbuf)) { + if (unlikely(!rxr->ag_buf_ring[i])) { if (bnxt_alloc_ag_data(rxq, rxr, prod) != 0) { PMD_DRV_LOG(WARNING, "init'ed AG ring %d with %d/%d mbufs only\n", diff --git a/drivers/net/bnxt/bnxt_rxr.h b/drivers/net/bnxt/bnxt_rxr.h index 2bf46cd910..5b9b5f3108 100644 --- a/drivers/net/bnxt/bnxt_rxr.h +++ b/drivers/net/bnxt/bnxt_rxr.h @@ -181,10 +181,6 @@ struct bnxt_tpa_info { struct rx_tpa_v2_abuf_cmpl agg_arr[TPA_MAX_NUM_SEGS]; }; -struct bnxt_sw_rx_bd { - struct rte_mbuf *mbuf; /* data associated with RX descriptor */ -}; - struct bnxt_rx_ring_info { uint16_t rx_prod; uint16_t ag_prod; @@ -194,8 +190,8 @@ struct bnxt_rx_ring_info { struct rx_prod_pkt_bd *rx_desc_ring; struct rx_prod_pkt_bd *ag_desc_ring; - struct bnxt_sw_rx_bd *rx_buf_ring; /* sw ring */ - struct bnxt_sw_rx_bd *ag_buf_ring; /* sw ring */ + struct rte_mbuf **rx_buf_ring; /* sw ring */ + struct rte_mbuf **ag_buf_ring; /* sw ring */ rte_iova_t rx_desc_mapping; rte_iova_t ag_desc_mapping; diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index bf76c2ac26..eff196f3a0 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -29,7 +29,7 @@ static inline void bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) { struct rx_prod_pkt_bd *rxbds = &rxr->rx_desc_ring[rxq->rxrearm_start]; - struct bnxt_sw_rx_bd *rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; + struct rte_mbuf **rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; struct rte_mbuf *mb0, *mb1; int i; @@ -51,8 +51,8 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) uint64x2_t buf_addr0, buf_addr1; uint64x2_t rxbd0, rxbd1; - mb0 = rx_bufs[0].mbuf; - mb1 = rx_bufs[1].mbuf; + mb0 = rx_bufs[0]; + mb1 = rx_bufs[1]; /* Load address fields from both mbufs */ buf_addr0 = vld1q_u64((uint64_t *)&mb0->buf_addr); @@ -260,9 +260,9 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, raw_cons = tmp_raw_cons; cons = rxcmp->opaque; - mbuf = rxr->rx_buf_ring[cons].mbuf; + mbuf = rxr->rx_buf_ring[cons]; rte_prefetch0(mbuf); - rxr->rx_buf_ring[cons].mbuf = NULL; + rxr->rx_buf_ring[cons] = NULL; /* Set constant fields from mbuf initializer. */ vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index 98220bc1b3..822e43343f 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -33,7 +33,7 @@ static inline void bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) { struct rx_prod_pkt_bd *rxbds = &rxr->rx_desc_ring[rxq->rxrearm_start]; - struct bnxt_sw_rx_bd *rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; + struct rte_mbuf **rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; struct rte_mbuf *mb0, *mb1; int i; @@ -55,8 +55,8 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) __m128i buf_addr0, buf_addr1; __m128i rxbd0, rxbd1; - mb0 = rx_bufs[0].mbuf; - mb1 = rx_bufs[1].mbuf; + mb0 = rx_bufs[0]; + mb1 = rx_bufs[1]; /* Load address fields from both mbufs */ buf_addr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); @@ -265,9 +265,9 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, raw_cons = tmp_raw_cons; cons = rxcmp->opaque; - mbuf = rxr->rx_buf_ring[cons].mbuf; + mbuf = rxr->rx_buf_ring[cons]; rte_prefetch0(mbuf); - rxr->rx_buf_ring[cons].mbuf = NULL; + rxr->rx_buf_ring[cons] = NULL; /* Set constant fields from mbuf initializer. */ _mm_store_si128((__m128i *)&mbuf->rearm_data, From patchwork Wed Sep 9 15:52:57 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77068 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6C069A04B5; Wed, 9 Sep 2020 17:53:57 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 3492E1C0D7; Wed, 9 Sep 2020 17:53:24 +0200 (CEST) Received: from mail-pj1-f66.google.com (mail-pj1-f66.google.com [209.85.216.66]) by dpdk.org (Postfix) with ESMTP id 3ECB71C10B for ; Wed, 9 Sep 2020 17:53:22 +0200 (CEST) Received: by mail-pj1-f66.google.com with SMTP id md22so1483058pjb.0 for ; Wed, 09 Sep 2020 08:53:22 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=y65K0o9d6Wnm++JqpdZXMeZmLP1Yy01AtF4NqRRv82o=; b=L0pdMSgHhh30Ub7C/a3+1FfAiJyjDJ8acwX8dL3kRNjd8+DDhltNOxdaWx9k9BgoW8 PG3Su4WD95TFhPRih1o3uKmlVy4hPjdqJzXDYKCQ/hKJaYbbrTEdPq1r4or4FA5T57cS yHTkb+stWO9un9BvigYrwmSM2/YMFz/mF5qEU= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=y65K0o9d6Wnm++JqpdZXMeZmLP1Yy01AtF4NqRRv82o=; b=RXDHIV5Iqtn0LpxH877iB6OX+vUz4YMuIATUy9HuaBWVelrECa4+gmr4Ulk9XL1vYQ oqMj2+cP4Dg4VMt3+wAcOY8ZRhNjI7E1KaNJIlcLvDm25n8uVxirv9GcJSHzCZ9PVgDS Ri+IZJfyOaMhRz+VDjl6STbxmjGKxphE8KRpTHQEl1Z4YtpbLDZBE/V8y8tPVogfnw2f V1GO4KbVXAKLmjp3UVwQLX40BeTvVBtd4klw22jaWLQnLTxjAqlQ+MAXCLQVwnc8zjF9 uuC8tUD/lLLgj15W+WjJvq9oAHQpZVE0u+/B3XS1vr4UvTaAMW8Aaw8QQHsSy1A+F5UP lMYg== X-Gm-Message-State: AOAM530CycvhMet5dVOelAgv8Q17vMKrQ1sTgft715vlZoECOiWZ9YKA lIJMhSF/OT3NxOwARC137WqTW3C629YzIA== X-Google-Smtp-Source: ABdhPJxa1bsexiUpNqWRRhAymow0QAg2iHdnWuL+vJGxc5YYsNJrK7hA52LHvb3lN7Tq6o+By+2xEg== X-Received: by 2002:a17:90b:4018:: with SMTP id ie24mr1394427pjb.9.1599666801167; Wed, 09 Sep 2020 08:53:21 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.20 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:20 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:52:57 -0400 Message-Id: <20200909155302.28656-5-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 04/12] net/bnxt: require async cq for vector mode X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Disable support for vector mode when async completions can be placed in a receive completion ring and change the default for all platforms to use a dedicated async completion ring. Simplify completion handling in vector mode receive paths now that it no longer needs to handle async completions. Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt.h | 19 ++-- drivers/net/bnxt/bnxt_ethdev.c | 2 +- drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 121 +++++++++++--------------- drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 116 +++++++++++------------- 4 files changed, 111 insertions(+), 147 deletions(-) diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h index a190d78bdd..ef5824cf9a 100644 --- a/drivers/net/bnxt/bnxt.h +++ b/drivers/net/bnxt/bnxt.h @@ -119,20 +119,19 @@ (BNXT_CHIP_THOR(bp) ? TPA_MAX_SEGS_TH : \ TPA_MAX_SEGS) -#ifdef RTE_ARCH_ARM64 -#define BNXT_NUM_ASYNC_CPR(bp) (BNXT_STINGRAY(bp) ? 0 : 1) +/* + * Define the number of async completion rings to be used. Set to zero for + * configurations in which the maximum number of packet completion rings + * for packet completions is desired or when async completion handling + * cannot be interrupt-driven. + */ +#ifdef RTE_EXEC_ENV_FREEBSD +/* In FreeBSD OS, nic_uio driver does not support interrupts */ +#define BNXT_NUM_ASYNC_CPR(bp) 0 #else #define BNXT_NUM_ASYNC_CPR(bp) 1 #endif -/* In FreeBSD OS, nic_uio driver does not support interrupts */ -#ifdef RTE_EXEC_ENV_FREEBSD -#ifdef BNXT_NUM_ASYNC_CPR -#undef BNXT_NUM_ASYNC_CPR -#endif -#define BNXT_NUM_ASYNC_CPR(bp) 0 -#endif - #define BNXT_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET #define BNXT_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index c57c5cc2af..1ad9bfc0a6 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -1114,7 +1114,7 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev) DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | DEV_RX_OFFLOAD_RSS_HASH | DEV_RX_OFFLOAD_VLAN_FILTER)) && - !BNXT_TRUFLOW_EN(bp)) { + !BNXT_TRUFLOW_EN(bp) && BNXT_NUM_ASYNC_CPR(bp)) { PMD_DRV_LOG(INFO, "Using vector mode receive for port %d\n", eth_dev->data->port_id); bp->flags |= BNXT_FLAG_RX_VECTOR_PKT_MODE; diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index eff196f3a0..a212d46cbe 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -206,7 +206,6 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint32_t cons; int nb_rx_pkts = 0; struct rx_pkt_cmpl *rxcmp; - bool evt = false; const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0}; const uint8x16_t shuf_msk = { 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type (zeroes) */ @@ -215,6 +214,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 0xFF, 0xFF, /* vlan_tci (zeroes) */ 12, 13, 14, 15 /* rss hash */ }; + int i; /* If Rx Q was stopped return */ if (unlikely(!rxq->rx_started)) @@ -226,90 +226,73 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, /* Return no more than RTE_BNXT_MAX_RX_BURST per call. */ nb_pkts = RTE_MIN(nb_pkts, RTE_BNXT_MAX_RX_BURST); - /* Make nb_pkts an integer multiple of RTE_BNXT_DESCS_PER_LOOP */ + /* Make nb_pkts an integer multiple of RTE_BNXT_DESCS_PER_LOOP. */ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_BNXT_DESCS_PER_LOOP); if (!nb_pkts) return 0; /* Handle RX burst request */ - while (1) { + for (i = 0; i < nb_pkts; i++) { + struct rx_pkt_cmpl_hi *rxcmp1; + struct rte_mbuf *mbuf; + uint64x2_t mm_rxcmp; + uint8x16_t pkt_mb; + cons = RING_CMP(cpr->cp_ring_struct, raw_cons); rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; + rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cons + 1]; - if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) + if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) break; - if (likely(CMP_TYPE(rxcmp) == RX_PKT_CMPL_TYPE_RX_L2)) { - struct rx_pkt_cmpl_hi *rxcmp1; - uint32_t tmp_raw_cons; - uint16_t cp_cons; - struct rte_mbuf *mbuf; - uint64x2_t mm_rxcmp; - uint8x16_t pkt_mb; - - tmp_raw_cons = NEXT_RAW_CMP(raw_cons); - cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); - rxcmp1 = (struct rx_pkt_cmpl_hi *) - &cpr->cp_desc_ring[cp_cons]; - - if (!CMP_VALID(rxcmp1, tmp_raw_cons, - cpr->cp_ring_struct)) - break; - - raw_cons = tmp_raw_cons; - cons = rxcmp->opaque; - - mbuf = rxr->rx_buf_ring[cons]; - rte_prefetch0(mbuf); - rxr->rx_buf_ring[cons] = NULL; - - /* Set constant fields from mbuf initializer. */ - vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); - - /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - mm_rxcmp = vld1q_u64((uint64_t *)rxcmp); - pkt_mb = vqtbl1q_u8(vreinterpretq_u8_u64(mm_rxcmp), - shuf_msk); - vst1q_u64((uint64_t *)&mbuf->rx_descriptor_fields1, - vreinterpretq_u64_u8(pkt_mb)); - - rte_compiler_barrier(); - - if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - mbuf->ol_flags |= PKT_RX_RSS_HASH; - - if (rxcmp1->flags2 & - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { - mbuf->vlan_tci = rxcmp1->metadata & - (RX_PKT_CMPL_METADATA_VID_MASK | - RX_PKT_CMPL_METADATA_DE | - RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= - PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; - } - - bnxt_parse_csum(mbuf, rxcmp1); - mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); - - rx_pkts[nb_rx_pkts++] = mbuf; - } else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) { - evt = - bnxt_event_hwrm_resp_handler(rxq->bp, - (struct cmpl_base *)rxcmp); + raw_cons += 2; + cons = rxcmp->opaque; + + mbuf = rxr->rx_buf_ring[cons]; + rte_prefetch0(mbuf); + rxr->rx_buf_ring[cons] = NULL; + + /* Set constant fields from mbuf initializer. */ + vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); + + /* Set mbuf pkt_len, data_len, and rss_hash fields. */ + mm_rxcmp = vld1q_u64((uint64_t *)rxcmp); + pkt_mb = vqtbl1q_u8(vreinterpretq_u8_u64(mm_rxcmp), shuf_msk); + vst1q_u64((uint64_t *)&mbuf->rx_descriptor_fields1, + vreinterpretq_u64_u8(pkt_mb)); + + rte_compiler_barrier(); + + if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) + mbuf->ol_flags |= PKT_RX_RSS_HASH; + + if (rxcmp1->flags2 & + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { + mbuf->vlan_tci = rxcmp1->metadata & + (RX_PKT_CMPL_METADATA_VID_MASK | + RX_PKT_CMPL_METADATA_DE | + RX_PKT_CMPL_METADATA_PRI_MASK); + mbuf->ol_flags |= + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; } - raw_cons = NEXT_RAW_CMP(raw_cons); - if (nb_rx_pkts == nb_pkts || evt) - break; + bnxt_parse_csum(mbuf, rxcmp1); + mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); + + rx_pkts[nb_rx_pkts++] = mbuf; } - rxr->rx_prod = RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); - rxq->rxrearm_nb += nb_rx_pkts; - cpr->cp_raw_cons = raw_cons; - cpr->valid = !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); - if (nb_rx_pkts || evt) + if (nb_rx_pkts) { + rxr->rx_prod = + RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); + + rxq->rxrearm_nb += nb_rx_pkts; + cpr->cp_raw_cons = raw_cons; + cpr->valid = + !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); bnxt_db_cq(cpr); + } return nb_rx_pkts; } diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index 822e43343f..c00d7f6807 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -210,7 +210,6 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint32_t cons; int nb_rx_pkts = 0; struct rx_pkt_cmpl *rxcmp; - bool evt = false; const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); const __m128i shuf_msk = _mm_set_epi8(15, 14, 13, 12, /* rss */ @@ -218,6 +217,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 3, 2, /* data_len */ 0xFF, 0xFF, 3, 2, /* pkt_len */ 0xFF, 0xFF, 0xFF, 0xFF); /* pkt_type (zeroes) */ + int i; /* If Rx Q was stopped return */ if (unlikely(!rxq->rx_started)) @@ -238,83 +238,65 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, return 0; /* Handle RX burst request */ - while (1) { + for (i = 0; i < nb_pkts; i++) { + struct rx_pkt_cmpl_hi *rxcmp1; + struct rte_mbuf *mbuf; + __m128i mm_rxcmp, pkt_mb; + cons = RING_CMP(cpr->cp_ring_struct, raw_cons); rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; + rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cons + 1]; - if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) + if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) break; - if (likely(CMP_TYPE(rxcmp) == RX_PKT_CMPL_TYPE_RX_L2)) { - struct rx_pkt_cmpl_hi *rxcmp1; - uint32_t tmp_raw_cons; - uint16_t cp_cons; - struct rte_mbuf *mbuf; - __m128i mm_rxcmp, pkt_mb; - - tmp_raw_cons = NEXT_RAW_CMP(raw_cons); - cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); - rxcmp1 = (struct rx_pkt_cmpl_hi *) - &cpr->cp_desc_ring[cp_cons]; - - if (!CMP_VALID(rxcmp1, tmp_raw_cons, - cpr->cp_ring_struct)) - break; - - raw_cons = tmp_raw_cons; - cons = rxcmp->opaque; - - mbuf = rxr->rx_buf_ring[cons]; - rte_prefetch0(mbuf); - rxr->rx_buf_ring[cons] = NULL; - - /* Set constant fields from mbuf initializer. */ - _mm_store_si128((__m128i *)&mbuf->rearm_data, - mbuf_init); - - /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - mm_rxcmp = _mm_load_si128((__m128i *)rxcmp); - pkt_mb = _mm_shuffle_epi8(mm_rxcmp, shuf_msk); - _mm_storeu_si128((void *)&mbuf->rx_descriptor_fields1, - pkt_mb); - - rte_compiler_barrier(); - - if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - mbuf->ol_flags |= PKT_RX_RSS_HASH; - - if (rxcmp1->flags2 & - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { - mbuf->vlan_tci = rxcmp1->metadata & - (RX_PKT_CMPL_METADATA_VID_MASK | - RX_PKT_CMPL_METADATA_DE | - RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= - PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; - } - - bnxt_parse_csum(mbuf, rxcmp1); - mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); - - rx_pkts[nb_rx_pkts++] = mbuf; - } else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) { - evt = - bnxt_event_hwrm_resp_handler(rxq->bp, - (struct cmpl_base *)rxcmp); + raw_cons += 2; + cons = rxcmp->opaque; + + mbuf = rxr->rx_buf_ring[cons]; + rte_prefetch0(mbuf); + rxr->rx_buf_ring[cons] = NULL; + + /* Set constant fields from mbuf initializer. */ + _mm_store_si128((__m128i *)&mbuf->rearm_data, mbuf_init); + + /* Set mbuf pkt_len, data_len, and rss_hash fields. */ + mm_rxcmp = _mm_load_si128((__m128i *)rxcmp); + pkt_mb = _mm_shuffle_epi8(mm_rxcmp, shuf_msk); + _mm_storeu_si128((void *)&mbuf->rx_descriptor_fields1, pkt_mb); + + rte_compiler_barrier(); + + if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) + mbuf->ol_flags |= PKT_RX_RSS_HASH; + + if (rxcmp1->flags2 & + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { + mbuf->vlan_tci = rxcmp1->metadata & + (RX_PKT_CMPL_METADATA_VID_MASK | + RX_PKT_CMPL_METADATA_DE | + RX_PKT_CMPL_METADATA_PRI_MASK); + mbuf->ol_flags |= + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; } - raw_cons = NEXT_RAW_CMP(raw_cons); - if (nb_rx_pkts == nb_pkts || evt) - break; + bnxt_parse_csum(mbuf, rxcmp1); + mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); + + rx_pkts[nb_rx_pkts++] = mbuf; } - rxr->rx_prod = RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); - rxq->rxrearm_nb += nb_rx_pkts; - cpr->cp_raw_cons = raw_cons; - cpr->valid = !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); - if (nb_rx_pkts || evt) + if (nb_rx_pkts) { + rxr->rx_prod = + RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); + + rxq->rxrearm_nb += nb_rx_pkts; + cpr->cp_raw_cons = raw_cons; + cpr->valid = + !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); bnxt_db_cq(cpr); + } return nb_rx_pkts; } From patchwork Wed Sep 9 15:52:58 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77069 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 3C303A04B5; Wed, 9 Sep 2020 17:54:11 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id B23111C126; Wed, 9 Sep 2020 17:53:25 +0200 (CEST) Received: from mail-pf1-f195.google.com (mail-pf1-f195.google.com [209.85.210.195]) by dpdk.org (Postfix) with ESMTP id A8CEA1C117 for ; Wed, 9 Sep 2020 17:53:24 +0200 (CEST) Received: by mail-pf1-f195.google.com with SMTP id x123so2522611pfc.7 for ; Wed, 09 Sep 2020 08:53:24 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=97XJfUL7PmpeZvsy0me8FV7Bf/NulqE5BzvLq9oKH7Y=; b=RFuR8w2MmsrCuMGF0JSi+0VdhDBzY5hIls/U3eTGjbocCcb7Kj0NwwoUlYRHK35Byg zvs2me9sfq6pI5eJKUnBvgSClILl4h07yE167b9L6MTByWNHI54vVnxjTxopBH5o8rGS XLX4GDVVyRU/Y9345BiPBM3SGHMizIS3JY7dY= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=97XJfUL7PmpeZvsy0me8FV7Bf/NulqE5BzvLq9oKH7Y=; b=p2pQDjFltbY//JCBYb6z5VB7Q+iZuucJcEfSaqQ2E00B7X5faOPiAqJ7AHKi/OUwBl EfHD+ZbP4iMTn/WSR95PztljVZmEYLwyu6lTaXasr5iM/HoQnmWqw+yxrIZUAzJHKiv9 oOxhcCJEA4DfyTF8U5OCXixOrVywtTPmr+5WwhJWDmZhGLH8WGVwWIbAdv5uk6hmI+ys RvIYwbjdsWbK+gyoU9cUau8u4Ymy1oZxBFfVtEfdONnnQencpmfV8tIIpLiIe6Aq0Pu9 jj10toIuaPAQCP/fh0JDv6f3XLUCu+5wAHlK0RcfSC4g5FGAXZQjCtxnNGeLj9UScTv9 I8JQ== X-Gm-Message-State: AOAM5323ndJq0xL16wdZuO/4IYRFgLhLLYE9mGCKaG1BkTtSvCtFBqwB IsEi07o3JAqKbgXRIBCVFVNSvw== X-Google-Smtp-Source: ABdhPJy36HbHpKRzYOMYzsglZaIgvur8f6kENCzW9rsMFjbIO2tcN9ojWLkV+Z6m892nTx3NFfN2lA== X-Received: by 2002:a17:902:a40e:: with SMTP id p14mr1383365plq.4.1599666803800; Wed, 09 Sep 2020 08:53:23 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.22 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:23 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:52:58 -0400 Message-Id: <20200909155302.28656-6-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 05/12] net/bnxt: improve support for small ring sizes X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Improve support for small ring sizes: - Ensure that transmit free threshold is no more than 1/4 ring size. - Ensure that receive free threshold is no more than 1/4 ring size. - Validate requested ring sizes against minimum supported size. - Use rxq receive free threshold instead of fixed maximum burst size to trigger bulk receive buffer allocation. Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_rxq.c | 6 +++-- drivers/net/bnxt/bnxt_rxtx_vec_common.h | 10 +++++---- drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 29 +++++++++++++------------ drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 29 +++++++++++++------------ drivers/net/bnxt/bnxt_txq.c | 7 ++++-- 5 files changed, 45 insertions(+), 36 deletions(-) diff --git a/drivers/net/bnxt/bnxt_rxq.c b/drivers/net/bnxt/bnxt_rxq.c index db9aa1f3ed..4ef3b5cb5c 100644 --- a/drivers/net/bnxt/bnxt_rxq.c +++ b/drivers/net/bnxt/bnxt_rxq.c @@ -14,6 +14,7 @@ #include "bnxt_rxq.h" #include "bnxt_rxr.h" #include "bnxt_vnic.h" +#include "bnxt_rxtx_vec_common.h" #include "hsi_struct_def_dpdk.h" /* @@ -305,7 +306,7 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, return -EINVAL; } - if (!nb_desc || nb_desc > MAX_RX_DESC_CNT) { + if (nb_desc < BNXT_MIN_RING_DESC || nb_desc > MAX_RX_DESC_CNT) { PMD_DRV_LOG(ERR, "nb_desc %d is invalid\n", nb_desc); rc = -EINVAL; goto out; @@ -326,7 +327,8 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, rxq->bp = bp; rxq->mb_pool = mp; rxq->nb_rx_desc = nb_desc; - rxq->rx_free_thresh = rx_conf->rx_free_thresh; + rxq->rx_free_thresh = + RTE_MIN(rte_align32pow2(nb_desc) / 4, RTE_BNXT_MAX_RX_BURST); PMD_DRV_LOG(DEBUG, "RX Buf MTU %d\n", eth_dev->data->mtu); diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_common.h b/drivers/net/bnxt/bnxt_rxtx_vec_common.h index 3da3c48f4e..2f28759d06 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_common.h +++ b/drivers/net/bnxt/bnxt_rxtx_vec_common.h @@ -5,11 +5,13 @@ #ifndef _BNXT_RXTX_VEC_COMMON_H_ #define _BNXT_RXTX_VEC_COMMON_H_ +#include "hsi_struct_def_dpdk.h" +#include "bnxt_rxq.h" +#include "bnxt_rxr.h" -#define RTE_BNXT_MAX_RX_BURST 32 -#define RTE_BNXT_MAX_TX_BURST 32 -#define RTE_BNXT_RXQ_REARM_THRESH 32 -#define RTE_BNXT_DESCS_PER_LOOP 4 +#define RTE_BNXT_MAX_RX_BURST 32U +#define RTE_BNXT_MAX_TX_BURST 32U +#define RTE_BNXT_DESCS_PER_LOOP 4U #define TX_BD_FLAGS_CMPL ((1 << TX_BD_LONG_FLAGS_BD_CNT_SFT) | \ TX_BD_SHORT_FLAGS_COAL_NOW | \ diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index a212d46cbe..7f3eabcda1 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -13,9 +13,6 @@ #include "bnxt.h" #include "bnxt_cpr.h" #include "bnxt_ring.h" -#include "bnxt_rxr.h" -#include "bnxt_rxq.h" -#include "hsi_struct_def_dpdk.h" #include "bnxt_rxtx_vec_common.h" #include "bnxt_txq.h" @@ -31,23 +28,27 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) struct rx_prod_pkt_bd *rxbds = &rxr->rx_desc_ring[rxq->rxrearm_start]; struct rte_mbuf **rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; struct rte_mbuf *mb0, *mb1; - int i; + int nb, i; const uint64x2_t hdr_room = {0, RTE_PKTMBUF_HEADROOM}; const uint64x2_t addrmask = {0, UINT64_MAX}; - /* Pull RTE_BNXT_RXQ_REARM_THRESH more mbufs into the software ring */ - if (rte_mempool_get_bulk(rxq->mb_pool, - (void *)rx_bufs, - RTE_BNXT_RXQ_REARM_THRESH) < 0) { - rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += - RTE_BNXT_RXQ_REARM_THRESH; + /* + * Number of mbufs to allocate must be a multiple of two. The + * allocation must not go past the end of the ring. + */ + nb = RTE_MIN(rxq->rxrearm_nb & ~0x1, + rxq->nb_rx_desc - rxq->rxrearm_start); + + /* Allocate new mbufs into the software ring */ + if (rte_mempool_get_bulk(rxq->mb_pool, (void *)rx_bufs, nb) < 0) { + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += nb; return; } /* Initialize the mbufs in vector, process 2 mbufs in one loop */ - for (i = 0; i < RTE_BNXT_RXQ_REARM_THRESH; i += 2, rx_bufs += 2) { + for (i = 0; i < nb; i += 2, rx_bufs += 2) { uint64x2_t buf_addr0, buf_addr1; uint64x2_t rxbd0, rxbd1; @@ -83,12 +84,12 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) vst1q_u64((uint64_t *)(rxbds++), rxbd1); } - rxq->rxrearm_start += RTE_BNXT_RXQ_REARM_THRESH; + rxq->rxrearm_start += nb; bnxt_db_write(&rxr->rx_db, rxq->rxrearm_start - 1); if (rxq->rxrearm_start >= rxq->nb_rx_desc) rxq->rxrearm_start = 0; - rxq->rxrearm_nb -= RTE_BNXT_RXQ_REARM_THRESH; + rxq->rxrearm_nb -= nb; } static uint32_t @@ -220,7 +221,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, if (unlikely(!rxq->rx_started)) return 0; - if (rxq->rxrearm_nb >= RTE_BNXT_RXQ_REARM_THRESH) + if (rxq->rxrearm_nb >= rxq->rx_free_thresh) bnxt_rxq_rearm(rxq, rxr); /* Return no more than RTE_BNXT_MAX_RX_BURST per call. */ diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index c00d7f6807..eced74e4e3 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -17,9 +17,6 @@ #include "bnxt.h" #include "bnxt_cpr.h" #include "bnxt_ring.h" -#include "bnxt_rxr.h" -#include "bnxt_rxq.h" -#include "hsi_struct_def_dpdk.h" #include "bnxt_rxtx_vec_common.h" #include "bnxt_txq.h" @@ -35,23 +32,27 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) struct rx_prod_pkt_bd *rxbds = &rxr->rx_desc_ring[rxq->rxrearm_start]; struct rte_mbuf **rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; struct rte_mbuf *mb0, *mb1; - int i; + int nb, i; const __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, 0); const __m128i addrmask = _mm_set_epi64x(UINT64_MAX, 0); - /* Pull RTE_BNXT_RXQ_REARM_THRESH more mbufs into the software ring */ - if (rte_mempool_get_bulk(rxq->mb_pool, - (void *)rx_bufs, - RTE_BNXT_RXQ_REARM_THRESH) < 0) { - rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += - RTE_BNXT_RXQ_REARM_THRESH; + /* + * Number of mbufs to allocate must be a multiple of two. The + * allocation must not go past the end of the ring. + */ + nb = RTE_MIN(rxq->rxrearm_nb & ~0x1, + rxq->nb_rx_desc - rxq->rxrearm_start); + + /* Allocate new mbufs into the software ring */ + if (rte_mempool_get_bulk(rxq->mb_pool, (void *)rx_bufs, nb) < 0) { + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += nb; return; } /* Initialize the mbufs in vector, process 2 mbufs in one loop */ - for (i = 0; i < RTE_BNXT_RXQ_REARM_THRESH; i += 2, rx_bufs += 2) { + for (i = 0; i < nb; i += 2, rx_bufs += 2) { __m128i buf_addr0, buf_addr1; __m128i rxbd0, rxbd1; @@ -87,12 +88,12 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) _mm_store_si128((__m128i *)(rxbds++), rxbd1); } - rxq->rxrearm_start += RTE_BNXT_RXQ_REARM_THRESH; + rxq->rxrearm_start += nb; bnxt_db_write(&rxr->rx_db, rxq->rxrearm_start - 1); if (rxq->rxrearm_start >= rxq->nb_rx_desc) rxq->rxrearm_start = 0; - rxq->rxrearm_nb -= RTE_BNXT_RXQ_REARM_THRESH; + rxq->rxrearm_nb -= nb; } static uint32_t @@ -223,7 +224,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, if (unlikely(!rxq->rx_started)) return 0; - if (rxq->rxrearm_nb >= RTE_BNXT_RXQ_REARM_THRESH) + if (rxq->rxrearm_nb >= rxq->rx_free_thresh) bnxt_rxq_rearm(rxq, rxr); /* Return no more than RTE_BNXT_MAX_RX_BURST per call. */ diff --git a/drivers/net/bnxt/bnxt_txq.c b/drivers/net/bnxt/bnxt_txq.c index 2d7645eeb0..42930abbf5 100644 --- a/drivers/net/bnxt/bnxt_txq.c +++ b/drivers/net/bnxt/bnxt_txq.c @@ -11,6 +11,7 @@ #include "bnxt_ring.h" #include "bnxt_txq.h" #include "bnxt_txr.h" +#include "bnxt_rxtx_vec_common.h" /* * TX Queues @@ -97,7 +98,7 @@ int bnxt_tx_queue_setup_op(struct rte_eth_dev *eth_dev, return -EINVAL; } - if (!nb_desc || nb_desc > MAX_TX_DESC_CNT) { + if (nb_desc < BNXT_MIN_RING_DESC || nb_desc > MAX_TX_DESC_CNT) { PMD_DRV_LOG(ERR, "nb_desc %d is invalid", nb_desc); rc = -EINVAL; goto out; @@ -129,7 +130,9 @@ int bnxt_tx_queue_setup_op(struct rte_eth_dev *eth_dev, } txq->bp = bp; txq->nb_tx_desc = nb_desc; - txq->tx_free_thresh = tx_conf->tx_free_thresh; + txq->tx_free_thresh = + RTE_MIN(rte_align32pow2(nb_desc) / 4, RTE_BNXT_MAX_TX_BURST); + txq->tx_deferred_start = tx_conf->tx_deferred_start; rc = bnxt_init_tx_ring_struct(txq, socket_id); From patchwork Wed Sep 9 15:52:59 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77070 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 19018A04B5; Wed, 9 Sep 2020 17:54:23 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 267DA1C12A; Wed, 9 Sep 2020 17:53:29 +0200 (CEST) Received: from mail-pf1-f196.google.com (mail-pf1-f196.google.com [209.85.210.196]) by dpdk.org (Postfix) with ESMTP id 0BBC21C12A for ; Wed, 9 Sep 2020 17:53:27 +0200 (CEST) Received: by mail-pf1-f196.google.com with SMTP id w7so2531543pfi.4 for ; Wed, 09 Sep 2020 08:53:26 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=64geMs9OE3TrwH5ljq2ZtZjNYVoSmHSQ1C/JxK521fw=; b=H3WwGnnwUuBFs56EI8LyOE3/pDSceUXUhve2syTsFAXru3GIPs1qVQYt8xyxla0oHY uYP833lYXslo3D52pkfcerkAYtFG7WeLKN3K9RRDAXS8//sjBoe1KOUQZnmHXZfI55YE /AE/7WBm5OHHGYLcz9l6sZSaAIuhbkBqGLrIY= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=64geMs9OE3TrwH5ljq2ZtZjNYVoSmHSQ1C/JxK521fw=; b=fKbCf20fyfzrNi9Uaih8lbOWtLQlgpSDDLQgX/fz4IHUIFaDALs2DKC3/R7ujHPqGi HzN0DVu2vIpkqaD3ma1n5sPOW8zFJz+LesoIwYvgZsbbS5mDm1jSuBaF6eiN+Yui76P1 U7QqldWeeN0hDf0243zqn0q5XhF5z6/Zrbydx2hwefbY++e1yy7mfeq/aLlv6qJadei1 kqsaYr2A1BphS5SWR/J+AaQlLGOHZJOL4rFhjLEM0gzOvgUg/qVqOJp2HOOcv7TnjjPy TRPm7pshqIsqoBZyfy7XvHie9Goo8uuxTTlBthwWSOskl9J1A6MIDNoBgM3OanKq8Di7 0XRg== X-Gm-Message-State: AOAM530i719AwtiIn5F/Asbvi1teL+zy8r8RX6YRkkQBeJqqZjDRFXtU cX/1AsgNm72280S6vnHpPSBnmg== X-Google-Smtp-Source: ABdhPJy4bJnwNOf1MrW6s3GvjjsUFIPAM/OSwy+wujOy+lU/vXFylBTRTRMGQrrLUw/Ezm0fGHRxEA== X-Received: by 2002:a62:2605:: with SMTP id m5mr1340217pfm.137.1599666806165; Wed, 09 Sep 2020 08:53:26 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.25 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:25 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:52:59 -0400 Message-Id: <20200909155302.28656-7-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 06/12] net/bnxt: use smaller cq when agg ring not needed X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Don't allocate extra completion queue entries for aggregation ring when aggregation ring will not be used. Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_ethdev.c | 11 +++++------ drivers/net/bnxt/bnxt_rxr.c | 21 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 1ad9bfc0a6..27eba431b8 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -1295,6 +1295,8 @@ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev) struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; eth_dev->data->dev_started = 0; + eth_dev->data->scattered_rx = 0; + /* Prevent crashes when queues are still in use */ eth_dev->rx_pkt_burst = &bnxt_dummy_recv_pkts; eth_dev->tx_pkt_burst = &bnxt_dummy_xmit_pkts; @@ -2695,14 +2697,12 @@ int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu) new_pkt_size = new_mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + VLAN_TAG_SIZE * BNXT_NUM_VLANS; -#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) /* - * If vector-mode tx/rx is active, disallow any MTU change that would - * require scattered receive support. + * Disallow any MTU change that would require scattered receive support + * if it is not already enabled. */ if (eth_dev->data->dev_started && - (eth_dev->rx_pkt_burst == bnxt_recv_pkts_vec || - eth_dev->tx_pkt_burst == bnxt_xmit_pkts_vec) && + !eth_dev->data->scattered_rx && (new_pkt_size > eth_dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { PMD_DRV_LOG(ERR, @@ -2710,7 +2710,6 @@ int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu) PMD_DRV_LOG(ERR, "Stop port before changing MTU.\n"); return -EINVAL; } -#endif if (new_mtu > RTE_ETHER_MTU) { bp->flags |= BNXT_FLAG_JUMBO; diff --git a/drivers/net/bnxt/bnxt_rxr.c b/drivers/net/bnxt/bnxt_rxr.c index 92102e3d57..5673e2b50f 100644 --- a/drivers/net/bnxt/bnxt_rxr.c +++ b/drivers/net/bnxt/bnxt_rxr.c @@ -938,9 +938,12 @@ void bnxt_free_rx_rings(struct bnxt *bp) int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) { + struct rte_eth_dev *eth_dev = rxq->bp->eth_dev; + struct rte_eth_rxmode *rxmode; struct bnxt_cp_ring_info *cpr; struct bnxt_rx_ring_info *rxr; struct bnxt_ring *ring; + bool use_agg_ring; rxq->rx_buf_size = BNXT_MAX_PKT_LEN + sizeof(struct rte_mbuf); @@ -978,8 +981,22 @@ int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) if (ring == NULL) return -ENOMEM; cpr->cp_ring_struct = ring; - ring->ring_size = rte_align32pow2(rxr->rx_ring_struct->ring_size * - (2 + AGG_RING_SIZE_FACTOR)); + + rxmode = ð_dev->data->dev_conf.rxmode; + use_agg_ring = (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER) || + (rxmode->offloads & DEV_RX_OFFLOAD_TCP_LRO) || + (rxmode->max_rx_pkt_len > + (uint32_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - + RTE_PKTMBUF_HEADROOM)); + + /* Allocate two completion slots per entry in desc ring. */ + ring->ring_size = rxr->rx_ring_struct->ring_size * 2; + + /* Allocate additional slots if aggregation ring is in use. */ + if (use_agg_ring) + ring->ring_size *= AGG_RING_SIZE_FACTOR; + + ring->ring_size = rte_align32pow2(ring->ring_size); ring->ring_mask = ring->ring_size - 1; ring->bd = (void *)cpr->cp_desc_ring; ring->bd_dma = cpr->cp_desc_mapping; From patchwork Wed Sep 9 15:53:00 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77071 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id DA685A04B5; Wed, 9 Sep 2020 17:54:32 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 79E451C133; Wed, 9 Sep 2020 17:53:31 +0200 (CEST) Received: from mail-pf1-f193.google.com (mail-pf1-f193.google.com [209.85.210.193]) by dpdk.org (Postfix) with ESMTP id 497381C12D for ; Wed, 9 Sep 2020 17:53:29 +0200 (CEST) Received: by mail-pf1-f193.google.com with SMTP id c196so2661229pfc.0 for ; Wed, 09 Sep 2020 08:53:29 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=Itv97QWp4ZpvXpe1R6x6gqTKMZX2hp4z6m6lmdqgv3A=; b=YEURXbNxXBpDgza9BPVvvUZsNdzv0MAGQyit8Qsqj4s3NHULH/LmDJEHy5fiPRYNgd abJJJcwaD5LlIC5EorAtj1DZIEyhhM23yn3bB6yvru32XFAo4P2+YW8WHETnu5AvEjEj 7AqClf+3OfEBEE+3jZbO8/fg/AeN5qQQsq6Bk= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=Itv97QWp4ZpvXpe1R6x6gqTKMZX2hp4z6m6lmdqgv3A=; b=CTiFhSIBw8Z42TEyKKGi63Ser+5rvD6xHKMqpeCk2eNsdsnLrEbB3lT89X7UlB9/TU pFWrWGngS6gyMIUamYtfB6aJspk/CG9XwTdEDY0972emO2MULE4Smfwt1K8Cdc/WCsZC knLeaVYM7JMpablrvRQPXTG64RKLswNPVAIxNJYGVDhVPPuU3ybgRfwAmqP6hAuKftSE SDz9YVL7GffDkJnb7a3rG4tBXF5d6uBImLe9IRWypbtDkfRA+znobLBBS7L1kwW1ik9o 3JjHPg4RgwFrkcAnXgwtCyD7cpk/+D2KZsN7Njn1YzR7P6YQlzbD+TOl0AnNOcb3R7ae /4fg== X-Gm-Message-State: AOAM533OfHTeTWaoBtxm5phs2NKehoy91DwFIFJNuKXiTFoprAShV71E UBnNvmgqd692WHrGtD3jW/+zLBVUHOoz0w== X-Google-Smtp-Source: ABdhPJyCBpSJPDE3APOabPVed4QKE456MLQcb8307qU+35UujGwfJWxGFAx3+T8nBmTsYxCHSiZOrQ== X-Received: by 2002:a63:1e4e:: with SMTP id p14mr1064843pgm.73.1599666808478; Wed, 09 Sep 2020 08:53:28 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.27 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:28 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org, Kalesh Anakkur Purayil Date: Wed, 9 Sep 2020 11:53:00 -0400 Message-Id: <20200909155302.28656-8-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 07/12] net/bnxt: increase max burst size for vector mode X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Increase the maximum supported burst size for the bnxt vector mode PMD from 32 to 64. Reviewed-by: Kalesh Anakkur Purayil Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_rxtx_vec_common.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_common.h b/drivers/net/bnxt/bnxt_rxtx_vec_common.h index 2f28759d06..fc2a12272b 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_common.h +++ b/drivers/net/bnxt/bnxt_rxtx_vec_common.h @@ -9,8 +9,8 @@ #include "bnxt_rxq.h" #include "bnxt_rxr.h" -#define RTE_BNXT_MAX_RX_BURST 32U -#define RTE_BNXT_MAX_TX_BURST 32U +#define RTE_BNXT_MAX_RX_BURST 64U +#define RTE_BNXT_MAX_TX_BURST 64U #define RTE_BNXT_DESCS_PER_LOOP 4U #define TX_BD_FLAGS_CMPL ((1 << TX_BD_LONG_FLAGS_BD_CNT_SFT) | \ From patchwork Wed Sep 9 15:53:01 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77072 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6DF82A04B5; Wed, 9 Sep 2020 17:54:43 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id CD87E1C138; Wed, 9 Sep 2020 17:53:32 +0200 (CEST) Received: from mail-pf1-f170.google.com (mail-pf1-f170.google.com [209.85.210.170]) by dpdk.org (Postfix) with ESMTP id CD0121C135 for ; Wed, 9 Sep 2020 17:53:31 +0200 (CEST) Received: by mail-pf1-f170.google.com with SMTP id k15so2502466pfc.12 for ; Wed, 09 Sep 2020 08:53:31 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=ifK5iZWa5OkCGGSVj77jmZUF5GvsafP/WFUinoqv1rw=; b=YIv2icgxAeK0tDWda3JtN1NYqRF9+lcg+pDUXbueM7XR6pIg977fJm+s3ClMyhnsbP kH6PftKUUzITwZTVHE0lcxrq9tlSt9GvLqM7md/cw58JI/9DttdOL496Sne/HPhMzeUD /jT5TqhQSwhP7C8ar6l8GfBU6ot6XIpdXOnhc= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=ifK5iZWa5OkCGGSVj77jmZUF5GvsafP/WFUinoqv1rw=; b=ly7VIfxvgt4l6urSQBD1BCkOu+4eim6KPVzMSEyT+Jikml1GyWjPYIY/dZ2vbu1IT+ Kl+L1TL69WFhyvBDyiskJb9EjXOJDmQ50U/tyPDvBaApP4GieGsVSMxhOmdgq5mTwA2C XjFHwyc8Z1cqOO9EXc7wbNN4b6Nbl6EDb+3WDVeK29zQZRxrwptKsxefL5D8YjP6M/UN JFqVWfdN7H4F6LXmZ+EG6lr8WjroUnqJ7GDjYymLeo6wK46vr+VR9aBu349gpFVpTPRi ElHjk4MOdUv/oajPg/e77+YGgthDWD3r63mr/sHB/npAz/smHq3w36UTTcjE+BOB0nTX pBtQ== X-Gm-Message-State: AOAM530u8E/0Atkfw8PNNtCoZK8Hc2fwsJakgO8u/xWw/+jVBiG6J1hG zh8dKqDnPg7AWYOquy3K20iN3g== X-Google-Smtp-Source: ABdhPJyFFSKTQCq7lVQX4EureDvto58J3KPBoEcp9TbbQQ0c2A9E9nXg2eCzGf2XolyvGCvuydZbCw== X-Received: by 2002:a63:242:: with SMTP id 63mr1070579pgc.182.1599666810821; Wed, 09 Sep 2020 08:53:30 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.29 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:30 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:53:01 -0400 Message-Id: <20200909155302.28656-9-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 08/12] net/bnxt: use table-based packet type translation X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Use table-based method for translating receive packet descriptor flags into rte_mbuf packet type values. Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_rxr.c | 127 ++++++++++++++++---------- drivers/net/bnxt/bnxt_rxr.h | 2 + drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 88 ++++++------------ drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 81 +++++----------- 4 files changed, 134 insertions(+), 164 deletions(-) diff --git a/drivers/net/bnxt/bnxt_rxr.c b/drivers/net/bnxt/bnxt_rxr.c index 5673e2b50f..a882dd20be 100644 --- a/drivers/net/bnxt/bnxt_rxr.c +++ b/drivers/net/bnxt/bnxt_rxr.c @@ -322,62 +322,88 @@ static inline struct rte_mbuf *bnxt_tpa_end( return mbuf; } -static uint32_t -bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) +uint32_t bnxt_ptype_table[BNXT_PTYPE_TBL_DIM] __rte_cache_aligned; + +static void __rte_cold +bnxt_init_ptype_table(void) { - uint32_t l3, pkt_type = 0; - uint32_t t_ipcs = 0, ip6 = 0, vlan = 0; - uint32_t flags_type; - - vlan = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN)); - pkt_type |= vlan ? RTE_PTYPE_L2_ETHER_VLAN : RTE_PTYPE_L2_ETHER; - - t_ipcs = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)); - ip6 = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_TYPE)); - - flags_type = rxcmp->flags_type & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS_ITYPE_MASK); - - if (!t_ipcs && !ip6) - l3 = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; - else if (!t_ipcs && ip6) - l3 = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; - else if (t_ipcs && !ip6) - l3 = RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; - else - l3 = RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; + uint32_t *pt = bnxt_ptype_table; + static bool initialized; + int ip6, tun, type; + uint32_t l3; + int i; - switch (flags_type) { - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_ICMP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_ICMP; - else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_ICMP; - break; + if (initialized) + return; - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_TCP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_TCP; + for (i = 0; i < BNXT_PTYPE_TBL_DIM; i++) { + if (i & (RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN >> 2)) + pt[i] = RTE_PTYPE_L2_ETHER_VLAN; else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_TCP; - break; - - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_UDP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_UDP; + pt[i] = RTE_PTYPE_L2_ETHER; + + ip6 = i & (RX_PKT_CMPL_FLAGS2_IP_TYPE >> 7); + tun = i & (RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC >> 2); + type = (i & 0x38) << 9; + + if (!tun && !ip6) + l3 = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + else if (!tun && ip6) + l3 = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + else if (tun && !ip6) + l3 = RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_UDP; - break; - - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_IP): - pkt_type |= l3; - break; + l3 = RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; + + switch (type) { + case RX_PKT_CMPL_FLAGS_ITYPE_ICMP: + if (tun) + pt[i] |= l3 | RTE_PTYPE_INNER_L4_ICMP; + else + pt[i] |= l3 | RTE_PTYPE_L4_ICMP; + break; + case RX_PKT_CMPL_FLAGS_ITYPE_TCP: + if (tun) + pt[i] |= l3 | RTE_PTYPE_INNER_L4_TCP; + else + pt[i] |= l3 | RTE_PTYPE_L4_TCP; + break; + case RX_PKT_CMPL_FLAGS_ITYPE_UDP: + if (tun) + pt[i] |= l3 | RTE_PTYPE_INNER_L4_UDP; + else + pt[i] |= l3 | RTE_PTYPE_L4_UDP; + break; + case RX_PKT_CMPL_FLAGS_ITYPE_IP: + pt[i] |= l3; + break; + } } + initialized = true; +} + +static uint32_t +bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) +{ + uint32_t flags_type, flags2; + uint8_t index; - return pkt_type; + flags_type = rte_le_to_cpu_16(rxcmp->flags_type); + flags2 = rte_le_to_cpu_32(rxcmp1->flags2); + + /* + * Index format: + * bit 0: RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC + * bit 1: RX_CMPL_FLAGS2_IP_TYPE + * bit 2: RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN + * bits 3-6: RX_PKT_CMPL_FLAGS_ITYPE + */ + index = ((flags_type & RX_PKT_CMPL_FLAGS_ITYPE_MASK) >> 9) | + ((flags2 & (RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)) >> 2) | + ((flags2 & RX_PKT_CMPL_FLAGS2_IP_TYPE) >> 7); + + return bnxt_ptype_table[index]; } #ifdef RTE_LIBRTE_IEEE1588 @@ -1046,6 +1072,9 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) unsigned int i; uint16_t size; + /* Initialize packet type table. */ + bnxt_init_ptype_table(); + size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; size = RTE_MIN(BNXT_MAX_PKT_LEN, size); diff --git a/drivers/net/bnxt/bnxt_rxr.h b/drivers/net/bnxt/bnxt_rxr.h index 5b9b5f3108..0e21c8f900 100644 --- a/drivers/net/bnxt/bnxt_rxr.h +++ b/drivers/net/bnxt/bnxt_rxr.h @@ -238,4 +238,6 @@ void bnxt_set_mark_in_mbuf(struct bnxt *bp, #define BNXT_CFA_META_EEM_TCAM_SHIFT 31 #define BNXT_CFA_META_EM_TEST(x) ((x) >> BNXT_CFA_META_EEM_TCAM_SHIFT) +#define BNXT_PTYPE_TBL_DIM 128 +extern uint32_t bnxt_ptype_table[BNXT_PTYPE_TBL_DIM]; #endif diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index 7f3eabcda1..fade67ec8e 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -93,61 +93,27 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) } static uint32_t -bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) +bnxt_parse_pkt_type(uint32x4_t mm_rxcmp, uint32x4_t mm_rxcmp1) { - uint32_t l3, pkt_type = 0; - uint32_t t_ipcs = 0, ip6 = 0, vlan = 0; - uint32_t flags_type; - - vlan = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN)); - pkt_type |= vlan ? RTE_PTYPE_L2_ETHER_VLAN : RTE_PTYPE_L2_ETHER; - - t_ipcs = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)); - ip6 = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_TYPE)); - - flags_type = rxcmp->flags_type & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS_ITYPE_MASK); - - if (!t_ipcs && !ip6) - l3 = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; - else if (!t_ipcs && ip6) - l3 = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; - else if (t_ipcs && !ip6) - l3 = RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; - else - l3 = RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; - - switch (flags_type) { - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_ICMP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_ICMP; - else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_ICMP; - break; + uint32_t flags_type, flags2; + uint8_t index; - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_TCP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_TCP; - else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_TCP; - break; + flags_type = vgetq_lane_u32(mm_rxcmp, 0); + flags2 = (uint16_t)vgetq_lane_u32(mm_rxcmp1, 0); - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_UDP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_UDP; - else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_UDP; - break; - - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_IP): - pkt_type |= l3; - break; - } + /* + * Index format: + * bit 0: RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC + * bit 1: RX_CMPL_FLAGS2_IP_TYPE + * bit 2: RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN + * bits 3-6: RX_PKT_CMPL_FLAGS_ITYPE + */ + index = ((flags_type & RX_PKT_CMPL_FLAGS_ITYPE_MASK) >> 9) | + ((flags2 & (RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)) >> 2) | + ((flags2 & RX_PKT_CMPL_FLAGS2_IP_TYPE) >> 7); - return pkt_type; + return bnxt_ptype_table[index]; } static void @@ -234,10 +200,12 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, /* Handle RX burst request */ for (i = 0; i < nb_pkts; i++) { + uint32x4_t mm_rxcmp, mm_rxcmp1; struct rx_pkt_cmpl_hi *rxcmp1; struct rte_mbuf *mbuf; - uint64x2_t mm_rxcmp; - uint8x16_t pkt_mb; + uint32x4_t pkt_mb; + uint8x16_t tmp; + uint32_t ptype; cons = RING_CMP(cpr->cp_ring_struct, raw_cons); @@ -247,6 +215,8 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) break; + mm_rxcmp = vld1q_u32((uint32_t *)rxcmp); + mm_rxcmp1 = vld1q_u32((uint32_t *)rxcmp); raw_cons += 2; cons = rxcmp->opaque; @@ -258,10 +228,12 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - mm_rxcmp = vld1q_u64((uint64_t *)rxcmp); - pkt_mb = vqtbl1q_u8(vreinterpretq_u8_u64(mm_rxcmp), shuf_msk); - vst1q_u64((uint64_t *)&mbuf->rx_descriptor_fields1, - vreinterpretq_u64_u8(pkt_mb)); + tmp = vqtbl1q_u8(vreinterpretq_u8_u32(mm_rxcmp), shuf_msk); + pkt_mb = vreinterpretq_u32_u8(tmp); + ptype = bnxt_parse_pkt_type(mm_rxcmp, mm_rxcmp1); + pkt_mb = vsetq_lane_u32(ptype, pkt_mb, 0); + + vst1q_u32((uint32_t *)&mbuf->rx_descriptor_fields1, pkt_mb); rte_compiler_barrier(); @@ -279,8 +251,6 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, } bnxt_parse_csum(mbuf, rxcmp1); - mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); - rx_pkts[nb_rx_pkts++] = mbuf; } diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index eced74e4e3..69ffbe4cc9 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -96,62 +96,28 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) rxq->rxrearm_nb -= nb; } -static uint32_t -bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) +static __m128i +bnxt_parse_pkt_type(__m128i mm_rxcmp, __m128i mm_rxcmp1) { - uint32_t l3, pkt_type = 0; - uint32_t t_ipcs = 0, ip6 = 0, vlan = 0; - uint32_t flags_type; - - vlan = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN)); - pkt_type |= vlan ? RTE_PTYPE_L2_ETHER_VLAN : RTE_PTYPE_L2_ETHER; - - t_ipcs = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)); - ip6 = !!(rxcmp1->flags2 & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_TYPE)); - - flags_type = rxcmp->flags_type & - rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS_ITYPE_MASK); - - if (!t_ipcs && !ip6) - l3 = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; - else if (!t_ipcs && ip6) - l3 = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; - else if (t_ipcs && !ip6) - l3 = RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; - else - l3 = RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; - - switch (flags_type) { - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_ICMP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_ICMP; - else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_ICMP; - break; - - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_TCP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_TCP; - else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_TCP; - break; + uint32_t flags_type, flags2; + uint8_t index; - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_UDP): - if (!t_ipcs) - pkt_type |= l3 | RTE_PTYPE_L4_UDP; - else - pkt_type |= l3 | RTE_PTYPE_INNER_L4_UDP; - break; + flags_type = _mm_extract_epi16(mm_rxcmp, 0); + flags2 = _mm_extract_epi32(mm_rxcmp1, 0); - case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_IP): - pkt_type |= l3; - break; - } + /* + * Index format: + * bit 0: RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC + * bit 1: RX_CMPL_FLAGS2_IP_TYPE + * bit 2: RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN + * bits 3-6: RX_PKT_CMPL_FLAGS_ITYPE + */ + index = ((flags_type & RX_PKT_CMPL_FLAGS_ITYPE_MASK) >> 9) | + ((flags2 & (RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)) >> 2) | + ((flags2 & RX_PKT_CMPL_FLAGS2_IP_TYPE) >> 7); - return pkt_type; + return _mm_set_epi32(0, 0, 0, bnxt_ptype_table[index]); } static void @@ -242,7 +208,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, for (i = 0; i < nb_pkts; i++) { struct rx_pkt_cmpl_hi *rxcmp1; struct rte_mbuf *mbuf; - __m128i mm_rxcmp, pkt_mb; + __m128i mm_rxcmp, mm_rxcmp1, pkt_mb, ptype; cons = RING_CMP(cpr->cp_ring_struct, raw_cons); @@ -252,6 +218,9 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) break; + mm_rxcmp = _mm_load_si128((__m128i *)rxcmp); + mm_rxcmp1 = _mm_load_si128((__m128i *)rxcmp1); + raw_cons += 2; cons = rxcmp->opaque; @@ -263,8 +232,10 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, _mm_store_si128((__m128i *)&mbuf->rearm_data, mbuf_init); /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - mm_rxcmp = _mm_load_si128((__m128i *)rxcmp); pkt_mb = _mm_shuffle_epi8(mm_rxcmp, shuf_msk); + ptype = bnxt_parse_pkt_type(mm_rxcmp, mm_rxcmp1); + pkt_mb = _mm_blend_epi16(pkt_mb, ptype, 0x3); + _mm_storeu_si128((void *)&mbuf->rx_descriptor_fields1, pkt_mb); rte_compiler_barrier(); @@ -283,8 +254,6 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, } bnxt_parse_csum(mbuf, rxcmp1); - mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); - rx_pkts[nb_rx_pkts++] = mbuf; } From patchwork Wed Sep 9 15:53:02 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77073 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 14B0CA04B5; Wed, 9 Sep 2020 17:54:57 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 676471C197; Wed, 9 Sep 2020 17:53:35 +0200 (CEST) Received: from mail-pf1-f196.google.com (mail-pf1-f196.google.com [209.85.210.196]) by dpdk.org (Postfix) with ESMTP id 1E6831C194 for ; Wed, 9 Sep 2020 17:53:34 +0200 (CEST) Received: by mail-pf1-f196.google.com with SMTP id k15so2502560pfc.12 for ; Wed, 09 Sep 2020 08:53:34 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=YQOZAO7EprWQq5D6jLRqslJ+oGqALExFy9CdnocF15U=; b=Dx65reN5LbcZL8Ki+nmxA+PX9s6ofoPSn9g1TyWgh+8NkbpfgMW1tdjMKQghPovxHQ A30u5PmeZAI44eyVj9Yh4oW3COkmWXSiWrySxlER65oeukClIpKlaVxoyEHfKdUL/a8q 52w8+hzKCpFlO+E8NS0cY8/QkGUb77KLvqq0o= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=YQOZAO7EprWQq5D6jLRqslJ+oGqALExFy9CdnocF15U=; b=edlDnfiS1B1J9ZlWf0wFlhxMMqpZB76oe8qfWkDWWs5xqfHzxqHRzmD4VjDNOVTXcy Pi57zDWXYvj6K0Ap0mZ6ILeRwZSgPVi+fM+QMQE8Lw8/usKimDCGfG0dnP6WKz6t/ghV QFJm695wUcZGX8+9+vWh21DYz5+wpRwn93m1VJtyRn2aQx5igXCIMT2x9oBPuMwBoT/L eFvla50qs0NELqphyBOWYSACzw29u21Tsn7gY43qMK/vnbNRTQXymA1PsTfKSxfMfOhS Ja01Bpq7LDRmpWK3ukITek52nzoB0QCmXHZpcBuoI228tRsG4Wfop3I0APZxCzvmT0L9 qS1w== X-Gm-Message-State: AOAM532+TEYMex+bPk6k9wsCeAG2Ro4w0cbgWSPFtPSLEPUqbIUhM7yW 7a16JFzDuEZHkesMOSarl2FzirlAsNRsZA== X-Google-Smtp-Source: ABdhPJxfovbsx/EVikafobqkoBMbSTp4kHFYx1WxUs8b9iJx491WXjFTvxwpW6ieZbwYiK2u+hPhxg== X-Received: by 2002:a62:5bc2:0:b029:13e:d13d:a090 with SMTP id p185-20020a625bc20000b029013ed13da090mr1383187pfb.39.1599666813152; Wed, 09 Sep 2020 08:53:33 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id h15sm3188427pfo.23.2020.09.09.08.53.32 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:53:32 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:53:02 -0400 Message-Id: <20200909155302.28656-10-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200909155302.28656-1-lance.richardson@broadcom.com> References: <20200909155302.28656-1-lance.richardson@broadcom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 09/12] net/bnxt: table-based handling for ol flags X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Use table to translate receive descriptor status flags to rte_mbuf ol_flags values. Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_rxr.c | 166 ++++++++++++++++---------- drivers/net/bnxt/bnxt_rxr.h | 6 + drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 99 ++++++--------- drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 96 ++++++--------- 4 files changed, 181 insertions(+), 186 deletions(-) diff --git a/drivers/net/bnxt/bnxt_rxr.c b/drivers/net/bnxt/bnxt_rxr.c index a882dd20be..33bd006530 100644 --- a/drivers/net/bnxt/bnxt_rxr.c +++ b/drivers/net/bnxt/bnxt_rxr.c @@ -406,6 +406,95 @@ bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) return bnxt_ptype_table[index]; } +uint32_t +bnxt_ol_flags_table[BNXT_OL_FLAGS_TBL_DIM] __rte_cache_aligned; + +uint32_t +bnxt_ol_flags_err_table[BNXT_OL_FLAGS_ERR_TBL_DIM] __rte_cache_aligned; + +static void __rte_cold +bnxt_init_ol_flags_tables(void) +{ + static bool initialized; + uint32_t *pt; + int i; + + if (initialized) + return; + + /* Initialize ol_flags table. */ + pt = bnxt_ol_flags_table; + for (i = 0; i < BNXT_OL_FLAGS_TBL_DIM; i++) { + pt[i] = 0; + if (i & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) + pt[i] |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; + + if (i & RX_PKT_CMPL_FLAGS2_IP_CS_CALC) + pt[i] |= PKT_RX_IP_CKSUM_GOOD; + + if (i & RX_PKT_CMPL_FLAGS2_L4_CS_CALC) + pt[i] |= PKT_RX_L4_CKSUM_GOOD; + + if (i & RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC) + pt[i] |= PKT_RX_OUTER_L4_CKSUM_GOOD; + } + + /* Initialize checksum error table. */ + pt = bnxt_ol_flags_err_table; + for (i = 0; i < BNXT_OL_FLAGS_ERR_TBL_DIM; i++) { + pt[i] = 0; + if (i & (RX_PKT_CMPL_ERRORS_IP_CS_ERROR >> 4)) + pt[i] |= PKT_RX_IP_CKSUM_BAD; + + if (i & (RX_PKT_CMPL_ERRORS_L4_CS_ERROR >> 4)) + pt[i] |= PKT_RX_L4_CKSUM_BAD; + + if (i & (RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR >> 4)) + pt[i] |= PKT_RX_EIP_CKSUM_BAD; + + if (i & (RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR >> 4)) + pt[i] |= PKT_RX_OUTER_L4_CKSUM_BAD; + } + + initialized = true; +} + +static void +bnxt_set_ol_flags(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1, + struct rte_mbuf *mbuf) +{ + uint16_t flags_type, errors, flags; + uint64_t ol_flags; + + flags_type = rte_le_to_cpu_16(rxcmp->flags_type); + + flags = rte_le_to_cpu_32(rxcmp1->flags2) & + (RX_PKT_CMPL_FLAGS2_IP_CS_CALC | + RX_PKT_CMPL_FLAGS2_L4_CS_CALC | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC | + RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC | + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN); + + errors = rte_le_to_cpu_16(rxcmp1->errors_v2) & + (RX_PKT_CMPL_ERRORS_IP_CS_ERROR | + RX_PKT_CMPL_ERRORS_L4_CS_ERROR | + RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR | + RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR); + errors = (errors >> 4) & flags; + + ol_flags = bnxt_ol_flags_table[flags & ~errors]; + + if (errors) + ol_flags |= bnxt_ol_flags_err_table[errors]; + + if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) { + mbuf->hash.rss = rte_le_to_cpu_32(rxcmp->rss_hash); + ol_flags |= PKT_RX_RSS_HASH; + } + + mbuf->ol_flags = ol_flags; +} + #ifdef RTE_LIBRTE_IEEE1588 static void bnxt_get_rx_ts_thor(struct bnxt *bp, uint32_t rx_ts_cmpl) @@ -583,8 +672,7 @@ static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, int rc = 0; uint8_t agg_buf = 0; uint16_t cmp_type; - uint32_t flags2_f = 0, vfr_flag = 0, mark_id = 0; - uint16_t flags_type; + uint32_t vfr_flag = 0, mark_id = 0; struct bnxt *bp = rxq->bp; rxcmp = (struct rx_pkt_cmpl *) @@ -653,13 +741,17 @@ static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, mbuf->pkt_len = rxcmp->len; mbuf->data_len = mbuf->pkt_len; mbuf->port = rxq->port_id; - mbuf->ol_flags = 0; - flags_type = rte_le_to_cpu_16(rxcmp->flags_type); - if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) { - mbuf->hash.rss = rxcmp->rss_hash; - mbuf->ol_flags |= PKT_RX_RSS_HASH; + bnxt_set_ol_flags(rxcmp, rxcmp1, mbuf); + +#ifdef RTE_LIBRTE_IEEE1588 + if (unlikely((rte_le_to_cpu_16(rxcmp->flags_type) & + RX_PKT_CMPL_FLAGS_MASK) == + RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP)) { + mbuf->ol_flags |= PKT_RX_IEEE1588_PTP | PKT_RX_IEEE1588_TMST; + bnxt_get_rx_ts_thor(rxq->bp, rxcmp1->reorder); } +#endif if (BNXT_TRUFLOW_EN(bp)) mark_id = bnxt_ulp_set_mark_in_mbuf(rxq->bp, rxcmp1, mbuf, @@ -667,66 +759,9 @@ static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, else bnxt_set_mark_in_mbuf(rxq->bp, rxcmp1, mbuf); -#ifdef RTE_LIBRTE_IEEE1588 - if (unlikely((flags_type & RX_PKT_CMPL_FLAGS_MASK) == - RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP)) { - mbuf->ol_flags |= PKT_RX_IEEE1588_PTP | PKT_RX_IEEE1588_TMST; - bnxt_get_rx_ts_thor(rxq->bp, rxcmp1->reorder); - } -#endif if (agg_buf) bnxt_rx_pages(rxq, mbuf, &tmp_raw_cons, agg_buf, NULL); - if (rxcmp1->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { - mbuf->vlan_tci = rxcmp1->metadata & - (RX_PKT_CMPL_METADATA_VID_MASK | - RX_PKT_CMPL_METADATA_DE | - RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; - } - - flags2_f = flags2_0xf(rxcmp1); - /* IP Checksum */ - if (likely(IS_IP_NONTUNNEL_PKT(flags2_f))) { - if (unlikely(RX_CMP_IP_CS_ERROR(rxcmp1))) - mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; - else if (unlikely(RX_CMP_IP_CS_UNKNOWN(rxcmp1))) - mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; - else - mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; - } else if (IS_IP_TUNNEL_PKT(flags2_f)) { - if (unlikely(RX_CMP_IP_OUTER_CS_ERROR(rxcmp1) || - RX_CMP_IP_CS_ERROR(rxcmp1))) - mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; - else if (unlikely(RX_CMP_IP_CS_UNKNOWN(rxcmp1))) - mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; - else - mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; - } - - /* L4 Checksum */ - if (likely(IS_L4_NONTUNNEL_PKT(flags2_f))) { - if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1))) - mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; - } else if (IS_L4_TUNNEL_PKT(flags2_f)) { - if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1))) - mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; - if (unlikely(RX_CMP_L4_OUTER_CS_ERR2(rxcmp1))) { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_BAD; - } else if (unlikely(IS_L4_TUNNEL_PKT_ONLY_INNER_L4_CS - (flags2_f))) { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_UNKNOWN; - } else { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_GOOD; - } - } else if (unlikely(RX_CMP_L4_CS_UNKNOWN(rxcmp1))) { - mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; - } - mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); #ifdef BNXT_DEBUG @@ -1075,6 +1110,9 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) /* Initialize packet type table. */ bnxt_init_ptype_table(); + /* Initialize offload flags parsing table. */ + bnxt_init_ol_flags_tables(); + size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; size = RTE_MIN(BNXT_MAX_PKT_LEN, size); diff --git a/drivers/net/bnxt/bnxt_rxr.h b/drivers/net/bnxt/bnxt_rxr.h index 0e21c8f900..4f5e23b855 100644 --- a/drivers/net/bnxt/bnxt_rxr.h +++ b/drivers/net/bnxt/bnxt_rxr.h @@ -240,4 +240,10 @@ void bnxt_set_mark_in_mbuf(struct bnxt *bp, #define BNXT_PTYPE_TBL_DIM 128 extern uint32_t bnxt_ptype_table[BNXT_PTYPE_TBL_DIM]; + +#define BNXT_OL_FLAGS_TBL_DIM 32 +extern uint32_t bnxt_ol_flags_table[BNXT_OL_FLAGS_TBL_DIM]; + +#define BNXT_OL_FLAGS_ERR_TBL_DIM 16 +extern uint32_t bnxt_ol_flags_err_table[BNXT_OL_FLAGS_ERR_TBL_DIM]; #endif diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index fade67ec8e..37b8c83656 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -116,50 +116,28 @@ bnxt_parse_pkt_type(uint32x4_t mm_rxcmp, uint32x4_t mm_rxcmp1) return bnxt_ptype_table[index]; } -static void -bnxt_parse_csum(struct rte_mbuf *mbuf, struct rx_pkt_cmpl_hi *rxcmp1) +static uint32_t +bnxt_set_ol_flags(uint32x4_t mm_rxcmp, uint32x4_t mm_rxcmp1) { - uint32_t flags; + uint16_t flags_type, errors, flags; + uint32_t ol_flags; - flags = flags2_0xf(rxcmp1); - /* IP Checksum */ - if (likely(IS_IP_NONTUNNEL_PKT(flags))) { - if (unlikely(RX_CMP_IP_CS_ERROR(rxcmp1))) - mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; - } else if (IS_IP_TUNNEL_PKT(flags)) { - if (unlikely(RX_CMP_IP_OUTER_CS_ERROR(rxcmp1) || - RX_CMP_IP_CS_ERROR(rxcmp1))) - mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; - } else if (unlikely(RX_CMP_IP_CS_UNKNOWN(rxcmp1))) { - mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; - } + /* Extract rxcmp1->flags2. */ + flags = vgetq_lane_u32(mm_rxcmp1, 0) & 0x1F; + /* Extract rxcmp->flags_type. */ + flags_type = vgetq_lane_u32(mm_rxcmp, 0); + /* Extract rxcmp1->errors_v2. */ + errors = (vgetq_lane_u32(mm_rxcmp1, 2) >> 4) & flags & 0xF; - /* L4 Checksum */ - if (likely(IS_L4_NONTUNNEL_PKT(flags))) { - if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1))) - mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; - } else if (IS_L4_TUNNEL_PKT(flags)) { - if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1))) - mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; - if (unlikely(RX_CMP_L4_OUTER_CS_ERR2(rxcmp1))) { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_BAD; - } else if (unlikely(IS_L4_TUNNEL_PKT_ONLY_INNER_L4_CS - (flags))) { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_UNKNOWN; - } else { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_GOOD; - } - } else if (unlikely(RX_CMP_L4_CS_UNKNOWN(rxcmp1))) { - mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; - } + ol_flags = bnxt_ol_flags_table[flags & ~errors]; + + if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) + ol_flags |= PKT_RX_RSS_HASH; + + if (errors) + ol_flags |= bnxt_ol_flags_err_table[errors]; + + return ol_flags; } uint16_t @@ -202,10 +180,12 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, for (i = 0; i < nb_pkts; i++) { uint32x4_t mm_rxcmp, mm_rxcmp1; struct rx_pkt_cmpl_hi *rxcmp1; + uint32x4_t pkt_mb, rearm; + uint32_t ptype, ol_flags; struct rte_mbuf *mbuf; - uint32x4_t pkt_mb; + uint16_t vlan_tci; + uint16x8_t tmp16; uint8x16_t tmp; - uint32_t ptype; cons = RING_CMP(cpr->cp_ring_struct, raw_cons); @@ -224,33 +204,30 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, rte_prefetch0(mbuf); rxr->rx_buf_ring[cons] = NULL; - /* Set constant fields from mbuf initializer. */ - vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); + /* Set fields from mbuf initializer and ol_flags. */ + ol_flags = bnxt_set_ol_flags(mm_rxcmp, mm_rxcmp1); + rearm = vsetq_lane_u32(ol_flags, + vreinterpretq_u32_u64(mbuf_init), 2); + vst1q_u32((uint32_t *)&mbuf->rearm_data, rearm); /* Set mbuf pkt_len, data_len, and rss_hash fields. */ tmp = vqtbl1q_u8(vreinterpretq_u8_u32(mm_rxcmp), shuf_msk); pkt_mb = vreinterpretq_u32_u8(tmp); + + /* Set packet type. */ ptype = bnxt_parse_pkt_type(mm_rxcmp, mm_rxcmp1); pkt_mb = vsetq_lane_u32(ptype, pkt_mb, 0); - vst1q_u32((uint32_t *)&mbuf->rx_descriptor_fields1, pkt_mb); + /* Set vlan_tci. */ + vlan_tci = vgetq_lane_u32(mm_rxcmp1, 1); + tmp16 = vsetq_lane_u16(vlan_tci, + vreinterpretq_u16_u32(pkt_mb), + 5); + pkt_mb = vreinterpretq_u32_u16(tmp16); - rte_compiler_barrier(); - - if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - mbuf->ol_flags |= PKT_RX_RSS_HASH; - - if (rxcmp1->flags2 & - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { - mbuf->vlan_tci = rxcmp1->metadata & - (RX_PKT_CMPL_METADATA_VID_MASK | - RX_PKT_CMPL_METADATA_DE | - RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= - PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; - } + /* Store descriptor fields. */ + vst1q_u32((uint32_t *)&mbuf->rx_descriptor_fields1, pkt_mb); - bnxt_parse_csum(mbuf, rxcmp1); rx_pkts[nb_rx_pkts++] = mbuf; } diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index 69ffbe4cc9..761d835963 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -120,50 +120,28 @@ bnxt_parse_pkt_type(__m128i mm_rxcmp, __m128i mm_rxcmp1) return _mm_set_epi32(0, 0, 0, bnxt_ptype_table[index]); } -static void -bnxt_parse_csum(struct rte_mbuf *mbuf, struct rx_pkt_cmpl_hi *rxcmp1) +static __m128i +bnxt_set_ol_flags(__m128i mm_rxcmp, __m128i mm_rxcmp1) { - uint32_t flags; + uint16_t flags_type, errors, flags; + uint32_t ol_flags; - flags = flags2_0xf(rxcmp1); - /* IP Checksum */ - if (likely(IS_IP_NONTUNNEL_PKT(flags))) { - if (unlikely(RX_CMP_IP_CS_ERROR(rxcmp1))) - mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; - } else if (IS_IP_TUNNEL_PKT(flags)) { - if (unlikely(RX_CMP_IP_OUTER_CS_ERROR(rxcmp1) || - RX_CMP_IP_CS_ERROR(rxcmp1))) - mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; - } else if (unlikely(RX_CMP_IP_CS_UNKNOWN(rxcmp1))) { - mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; - } + /* Extract rxcmp1->flags2. */ + flags = _mm_extract_epi32(mm_rxcmp1, 0) & 0x1F; + /* Extract rxcmp->flags_type. */ + flags_type = _mm_extract_epi16(mm_rxcmp, 0); + /* Extract rxcmp1->errors_v2. */ + errors = (_mm_extract_epi16(mm_rxcmp1, 4) >> 4) & flags & 0xF; - /* L4 Checksum */ - if (likely(IS_L4_NONTUNNEL_PKT(flags))) { - if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1))) - mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; - } else if (IS_L4_TUNNEL_PKT(flags)) { - if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1))) - mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; - else - mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; - if (unlikely(RX_CMP_L4_OUTER_CS_ERR2(rxcmp1))) { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_BAD; - } else if (unlikely(IS_L4_TUNNEL_PKT_ONLY_INNER_L4_CS - (flags))) { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_UNKNOWN; - } else { - mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_GOOD; - } - } else if (unlikely(RX_CMP_L4_CS_UNKNOWN(rxcmp1))) { - mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; - } + ol_flags = bnxt_ol_flags_table[flags & ~errors]; + + if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) + ol_flags |= PKT_RX_RSS_HASH; + + if (errors) + ol_flags |= bnxt_ol_flags_err_table[errors]; + + return _mm_set_epi64x(ol_flags, 0); } uint16_t @@ -208,7 +186,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, for (i = 0; i < nb_pkts; i++) { struct rx_pkt_cmpl_hi *rxcmp1; struct rte_mbuf *mbuf; - __m128i mm_rxcmp, mm_rxcmp1, pkt_mb, ptype; + __m128i mm_rxcmp, mm_rxcmp1, pkt_mb, ptype, rearm; cons = RING_CMP(cpr->cp_ring_struct, raw_cons); @@ -225,35 +203,31 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, cons = rxcmp->opaque; mbuf = rxr->rx_buf_ring[cons]; - rte_prefetch0(mbuf); rxr->rx_buf_ring[cons] = NULL; - /* Set constant fields from mbuf initializer. */ - _mm_store_si128((__m128i *)&mbuf->rearm_data, mbuf_init); + /* Set fields from mbuf initializer and ol_flags. */ + rearm = _mm_or_si128(mbuf_init, + bnxt_set_ol_flags(mm_rxcmp, mm_rxcmp1)); + _mm_store_si128((__m128i *)&mbuf->rearm_data, rearm); /* Set mbuf pkt_len, data_len, and rss_hash fields. */ pkt_mb = _mm_shuffle_epi8(mm_rxcmp, shuf_msk); + + /* Set packet type. */ ptype = bnxt_parse_pkt_type(mm_rxcmp, mm_rxcmp1); pkt_mb = _mm_blend_epi16(pkt_mb, ptype, 0x3); - _mm_storeu_si128((void *)&mbuf->rx_descriptor_fields1, pkt_mb); + /* + * Shift vlan_tci from completion metadata field left six + * bytes and blend into mbuf->rx_descriptor_fields1 to set + * mbuf->vlan_tci. + */ + pkt_mb = _mm_blend_epi16(pkt_mb, + _mm_slli_si128(mm_rxcmp1, 6), 0x20); - rte_compiler_barrier(); - - if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - mbuf->ol_flags |= PKT_RX_RSS_HASH; - - if (rxcmp1->flags2 & - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { - mbuf->vlan_tci = rxcmp1->metadata & - (RX_PKT_CMPL_METADATA_VID_MASK | - RX_PKT_CMPL_METADATA_DE | - RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= - PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; - } + /* Store descriptor fields. */ + _mm_storeu_si128((void *)&mbuf->rx_descriptor_fields1, pkt_mb); - bnxt_parse_csum(mbuf, rxcmp1); rx_pkts[nb_rx_pkts++] = mbuf; } From patchwork Wed Sep 9 15:57:00 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77074 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 7DABDA04B5; Wed, 9 Sep 2020 17:57:07 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 639571C0CA; Wed, 9 Sep 2020 17:57:07 +0200 (CEST) Received: from mail-pj1-f66.google.com (mail-pj1-f66.google.com [209.85.216.66]) by dpdk.org (Postfix) with ESMTP id 498D6DE0 for ; Wed, 9 Sep 2020 17:57:06 +0200 (CEST) Received: by mail-pj1-f66.google.com with SMTP id gf14so1557626pjb.5 for ; Wed, 09 Sep 2020 08:57:06 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=Z0wuH47Dp7l+9fYBEMjXWrPerQL8vAgUKcxWfTvRD84=; b=PVs1VF+60JCCNFA9HmxaoP+bNTQylv6ifLt8wPtWq5QqHJJRKWtHHreuu6nx8T1u6E PrrAgxcnISJRBhVKg5hTTRDXDLI2m8C8YwtOKDjWGKkg6qmvinsrTyADVwwJpbjZ842e +MHeDHjIotlY0APiHHoGn/llk2hfcFVpIffSE= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=Z0wuH47Dp7l+9fYBEMjXWrPerQL8vAgUKcxWfTvRD84=; b=BpOAr7KoQSwDafo2kjLg8nf0GklOBrgX0mpfXU5BSeh/5SHi7hkzwwlEG/SVnq410c 3W0fekPAt1BztgIwT/ySu9uylH24O0jqvvIdheKuvWz1xwnbVZMCKViLibcfSTTDanhv 3d6EXTuOxJW8l3eSUhIKYg6UfdMlBb8eZG8x6/fnCQ6BAnzHgjsMEJJZkXOIO3Ai+FgT WTXvkimmjxMNbtWJ/ZGtYtgpc/6Mh7qc7Z9UPg6OtkzroDig5nllnRuyuafDfN1lno30 hmucYX6X2K/kafOjMlJISFBzn0CukqffFtru4S7i6aDCRjJfqHe/jLJ78492yW+8P3bf /MgA== X-Gm-Message-State: AOAM533yUwWnbidWmk4+GD5MK1BI5ZlcbVi4j8fpCTrsPpEHW1Y9BcqK BaIqi7csqw/BOJj5bOxuz/wrHw== X-Google-Smtp-Source: ABdhPJzZt+V1j6cfB4xFnaOnuCRCl7SEXYIIaW7NjNBAJAQaUM3O9XEW19cVHR1xo2+gqsn1P2YxjA== X-Received: by 2002:a17:90a:a081:: with SMTP id r1mr1342421pjp.115.1599667025343; Wed, 09 Sep 2020 08:57:05 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id y128sm3025107pfy.74.2020.09.09.08.57.04 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:57:04 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:57:00 -0400 Message-Id: <20200909155700.29016-1-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 10/12] net/bnxt: optimize vector mode mbuf allocation X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Simplify and optimize receive mbuf allocation function used by the vector mode PMDs. Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_rxtx_vec_common.h | 40 ++++++++++++++ drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 70 ------------------------- drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 70 ------------------------- 3 files changed, 40 insertions(+), 140 deletions(-) diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_common.h b/drivers/net/bnxt/bnxt_rxtx_vec_common.h index fc2a12272b..819b8290e4 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_common.h +++ b/drivers/net/bnxt/bnxt_rxtx_vec_common.h @@ -56,4 +56,44 @@ bnxt_rxq_vec_setup_common(struct bnxt_rx_queue *rxq) rxq->rxrearm_start = 0; return 0; } + +static inline void +bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) +{ + struct rx_prod_pkt_bd *rxbds = &rxr->rx_desc_ring[rxq->rxrearm_start]; + struct rte_mbuf **rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; + int nb, i; + + /* + * Number of mbufs to allocate must be a multiple of four. The + * allocation must not go past the end of the ring. + */ + nb = RTE_MIN(rxq->rxrearm_nb & ~0x3, + rxq->nb_rx_desc - rxq->rxrearm_start); + + /* Allocate new mbufs into the software ring. */ + if (rte_mempool_get_bulk(rxq->mb_pool, (void *)rx_bufs, nb) < 0) { + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += nb; + + return; + } + + /* Initialize the mbufs in vector, process 4 mbufs per loop. */ + for (i = 0; i < nb; i += 4) { + rxbds[0].address = rte_mbuf_data_iova_default(rx_bufs[0]); + rxbds[1].address = rte_mbuf_data_iova_default(rx_bufs[1]); + rxbds[2].address = rte_mbuf_data_iova_default(rx_bufs[2]); + rxbds[3].address = rte_mbuf_data_iova_default(rx_bufs[3]); + + rxbds += 4; + rx_bufs += 4; + } + + rxq->rxrearm_start += nb; + bnxt_db_write(&rxr->rx_db, rxq->rxrearm_start - 1); + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= nb; +} #endif /* _BNXT_RXTX_VEC_COMMON_H_ */ diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index 37b8c83656..24f9fc3c39 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -22,76 +22,6 @@ * RX Ring handling */ -static inline void -bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) -{ - struct rx_prod_pkt_bd *rxbds = &rxr->rx_desc_ring[rxq->rxrearm_start]; - struct rte_mbuf **rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; - struct rte_mbuf *mb0, *mb1; - int nb, i; - - const uint64x2_t hdr_room = {0, RTE_PKTMBUF_HEADROOM}; - const uint64x2_t addrmask = {0, UINT64_MAX}; - - /* - * Number of mbufs to allocate must be a multiple of two. The - * allocation must not go past the end of the ring. - */ - nb = RTE_MIN(rxq->rxrearm_nb & ~0x1, - rxq->nb_rx_desc - rxq->rxrearm_start); - - /* Allocate new mbufs into the software ring */ - if (rte_mempool_get_bulk(rxq->mb_pool, (void *)rx_bufs, nb) < 0) { - rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += nb; - - return; - } - - /* Initialize the mbufs in vector, process 2 mbufs in one loop */ - for (i = 0; i < nb; i += 2, rx_bufs += 2) { - uint64x2_t buf_addr0, buf_addr1; - uint64x2_t rxbd0, rxbd1; - - mb0 = rx_bufs[0]; - mb1 = rx_bufs[1]; - - /* Load address fields from both mbufs */ - buf_addr0 = vld1q_u64((uint64_t *)&mb0->buf_addr); - buf_addr1 = vld1q_u64((uint64_t *)&mb1->buf_addr); - - /* Load both rx descriptors (preserving some existing fields) */ - rxbd0 = vld1q_u64((uint64_t *)(rxbds + 0)); - rxbd1 = vld1q_u64((uint64_t *)(rxbds + 1)); - - /* Add default offset to buffer address. */ - buf_addr0 = vaddq_u64(buf_addr0, hdr_room); - buf_addr1 = vaddq_u64(buf_addr1, hdr_room); - - /* Clear all fields except address. */ - buf_addr0 = vandq_u64(buf_addr0, addrmask); - buf_addr1 = vandq_u64(buf_addr1, addrmask); - - /* Clear address field in descriptor. */ - rxbd0 = vbicq_u64(rxbd0, addrmask); - rxbd1 = vbicq_u64(rxbd1, addrmask); - - /* Set address field in descriptor. */ - rxbd0 = vaddq_u64(rxbd0, buf_addr0); - rxbd1 = vaddq_u64(rxbd1, buf_addr1); - - /* Store descriptors to memory. */ - vst1q_u64((uint64_t *)(rxbds++), rxbd0); - vst1q_u64((uint64_t *)(rxbds++), rxbd1); - } - - rxq->rxrearm_start += nb; - bnxt_db_write(&rxr->rx_db, rxq->rxrearm_start - 1); - if (rxq->rxrearm_start >= rxq->nb_rx_desc) - rxq->rxrearm_start = 0; - - rxq->rxrearm_nb -= nb; -} - static uint32_t bnxt_parse_pkt_type(uint32x4_t mm_rxcmp, uint32x4_t mm_rxcmp1) { diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index 761d835963..7e87555408 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -26,76 +26,6 @@ * RX Ring handling */ -static inline void -bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) -{ - struct rx_prod_pkt_bd *rxbds = &rxr->rx_desc_ring[rxq->rxrearm_start]; - struct rte_mbuf **rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; - struct rte_mbuf *mb0, *mb1; - int nb, i; - - const __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, 0); - const __m128i addrmask = _mm_set_epi64x(UINT64_MAX, 0); - - /* - * Number of mbufs to allocate must be a multiple of two. The - * allocation must not go past the end of the ring. - */ - nb = RTE_MIN(rxq->rxrearm_nb & ~0x1, - rxq->nb_rx_desc - rxq->rxrearm_start); - - /* Allocate new mbufs into the software ring */ - if (rte_mempool_get_bulk(rxq->mb_pool, (void *)rx_bufs, nb) < 0) { - rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += nb; - - return; - } - - /* Initialize the mbufs in vector, process 2 mbufs in one loop */ - for (i = 0; i < nb; i += 2, rx_bufs += 2) { - __m128i buf_addr0, buf_addr1; - __m128i rxbd0, rxbd1; - - mb0 = rx_bufs[0]; - mb1 = rx_bufs[1]; - - /* Load address fields from both mbufs */ - buf_addr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); - buf_addr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); - - /* Load both rx descriptors (preserving some existing fields) */ - rxbd0 = _mm_loadu_si128((__m128i *)(rxbds + 0)); - rxbd1 = _mm_loadu_si128((__m128i *)(rxbds + 1)); - - /* Add default offset to buffer address. */ - buf_addr0 = _mm_add_epi64(buf_addr0, hdr_room); - buf_addr1 = _mm_add_epi64(buf_addr1, hdr_room); - - /* Clear all fields except address. */ - buf_addr0 = _mm_and_si128(buf_addr0, addrmask); - buf_addr1 = _mm_and_si128(buf_addr1, addrmask); - - /* Clear address field in descriptor. */ - rxbd0 = _mm_andnot_si128(addrmask, rxbd0); - rxbd1 = _mm_andnot_si128(addrmask, rxbd1); - - /* Set address field in descriptor. */ - rxbd0 = _mm_add_epi64(rxbd0, buf_addr0); - rxbd1 = _mm_add_epi64(rxbd1, buf_addr1); - - /* Store descriptors to memory. */ - _mm_store_si128((__m128i *)(rxbds++), rxbd0); - _mm_store_si128((__m128i *)(rxbds++), rxbd1); - } - - rxq->rxrearm_start += nb; - bnxt_db_write(&rxr->rx_db, rxq->rxrearm_start - 1); - if (rxq->rxrearm_start >= rxq->nb_rx_desc) - rxq->rxrearm_start = 0; - - rxq->rxrearm_nb -= nb; -} - static __m128i bnxt_parse_pkt_type(__m128i mm_rxcmp, __m128i mm_rxcmp1) { From patchwork Wed Sep 9 15:57:17 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77075 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E15CBA04B5; Wed, 9 Sep 2020 17:57:25 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id CE95F1C0DC; Wed, 9 Sep 2020 17:57:25 +0200 (CEST) Received: from mail-pg1-f196.google.com (mail-pg1-f196.google.com [209.85.215.196]) by dpdk.org (Postfix) with ESMTP id 15B901C0DA for ; Wed, 9 Sep 2020 17:57:24 +0200 (CEST) Received: by mail-pg1-f196.google.com with SMTP id 7so2359491pgm.11 for ; Wed, 09 Sep 2020 08:57:24 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=iEa4pKUyi4nAEsrKZiz6m8MkzSEuBTQZLK7Kr9pTa8c=; b=aZX06WtOtk59hEmniiB2/tu8fqqyad6tTQH74ld/KCQRj7CNolpHCl/eNOFIKw+4cn eAwMJpHhCE6XPk4TCAm6LU7p60k3gjVgucxWNs5H0S+eKVeZ4lRjxRJ3oNu959o4rAFI nIiU8+pDe0Cr2bZRLxsbG/lR7ZyrXodthAcPo= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=iEa4pKUyi4nAEsrKZiz6m8MkzSEuBTQZLK7Kr9pTa8c=; b=a/hnegoXFocK1AjRJhNVyU1Tyu3xXZWq0SwtoNbsfY2+i2OHtg5iDull4X186n0Y3c KY23sRdhWKU5OxHbrqmi/T2r07mlMhGh0bJtjMdMNuzxaaEzRyaH0MdKjny25f7z8Vn9 dok5vhtDCQmzvVrZuGMCzbD+8GzDkoWfB0YsRcEZH4BZ6mPjZmH0A9xx14hBiRsYq5lf ZWIFJMb3HbvdOJ3G+QJIYvPUIGbTMkMPxzmBmRdgEjR5j+F9lj0uuXUuoFozMHgeXPeX Gq4vTIotj2CSEX8lmQMbDLeExrWFl02NYKVS2dVtXN7+lMafOkN3+01jfoh8sxZP4NWL FA+g== X-Gm-Message-State: AOAM5311cGyIsSCdP13okYgYWCwwguFhZEYzy16xYzefCqT2yHnfBGGW +YmJ95TSicNTKpur+BplN2mNzA== X-Google-Smtp-Source: ABdhPJwJ1nZlWOqIeN1gvybJKm8I8970QXK34O7ZShpN2yEyuuC06KSQI8Ctfg9oqNcQza85brG/ew== X-Received: by 2002:a17:902:aa8d:: with SMTP id d13mr1446483plr.124.1599667042811; Wed, 09 Sep 2020 08:57:22 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id l7sm2320480pjz.56.2020.09.09.08.57.21 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:57:22 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:57:17 -0400 Message-Id: <20200909155717.29099-1-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 11/12] net/bnxt: handle multiple packets per loop in vector PMD X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Process four receive descriptors per inner loop in vector mode burst receive functions. Reviewed-by: Ajit Kumar Khaparde Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_ethdev.c | 2 +- drivers/net/bnxt/bnxt_rxq.c | 3 +- drivers/net/bnxt/bnxt_rxq.h | 1 + drivers/net/bnxt/bnxt_rxr.c | 15 +- drivers/net/bnxt/bnxt_rxtx_vec_common.h | 2 + drivers/net/bnxt/bnxt_rxtx_vec_neon.c | 365 +++++++++++++++++------- drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 316 ++++++++++++++------ 7 files changed, 508 insertions(+), 196 deletions(-) diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 27eba431b8..b658a44303 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -2872,7 +2872,7 @@ bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset) return RTE_ETH_RX_DESC_DONE; } rx_buf = rxr->rx_buf_ring[cons]; - if (rx_buf == NULL) + if (rx_buf == NULL || rx_buf == &rxq->fake_mbuf) return RTE_ETH_RX_DESC_UNAVAIL; diff --git a/drivers/net/bnxt/bnxt_rxq.c b/drivers/net/bnxt/bnxt_rxq.c index 4ef3b5cb5c..57ba9a1570 100644 --- a/drivers/net/bnxt/bnxt_rxq.c +++ b/drivers/net/bnxt/bnxt_rxq.c @@ -212,7 +212,8 @@ void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) for (i = 0; i < rxq->rx_ring->rx_ring_struct->ring_size; i++) { if (sw_ring[i]) { - rte_pktmbuf_free_seg(sw_ring[i]); + if (sw_ring[i] != &rxq->fake_mbuf) + rte_pktmbuf_free_seg(sw_ring[i]); sw_ring[i] = NULL; } } diff --git a/drivers/net/bnxt/bnxt_rxq.h b/drivers/net/bnxt/bnxt_rxq.h index d5ce3b6d58..96c6e06a52 100644 --- a/drivers/net/bnxt/bnxt_rxq.h +++ b/drivers/net/bnxt/bnxt_rxq.h @@ -39,6 +39,7 @@ struct bnxt_rx_queue { uint32_t rx_buf_size; struct bnxt_rx_ring_info *rx_ring; struct bnxt_cp_ring_info *cp_ring; + struct rte_mbuf fake_mbuf; rte_atomic64_t rx_mbuf_alloc_fail; const struct rte_memzone *mz; }; diff --git a/drivers/net/bnxt/bnxt_rxr.c b/drivers/net/bnxt/bnxt_rxr.c index 33bd006530..89a964a49b 100644 --- a/drivers/net/bnxt/bnxt_rxr.c +++ b/drivers/net/bnxt/bnxt_rxr.c @@ -20,6 +20,7 @@ #ifdef RTE_LIBRTE_IEEE1588 #include "bnxt_hwrm.h" #endif +#include "bnxt_rxtx_vec_common.h" #include #include @@ -931,7 +932,7 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, struct rte_mbuf **rx_buf = &rxr->rx_buf_ring[i]; /* Buffer already allocated for this index. */ - if (*rx_buf != NULL) + if (*rx_buf != NULL && *rx_buf != &rxq->fake_mbuf) continue; /* This slot is empty. Alloc buffer for Rx */ @@ -1025,7 +1026,11 @@ int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) ring->ring_mask = ring->ring_size - 1; ring->bd = (void *)rxr->rx_desc_ring; ring->bd_dma = rxr->rx_desc_mapping; - ring->vmem_size = ring->ring_size * sizeof(struct rte_mbuf *); + + /* Allocate extra rx ring entries for vector rx. */ + ring->vmem_size = sizeof(struct rte_mbuf *) * + (ring->ring_size + RTE_BNXT_DESCS_PER_LOOP); + ring->vmem = (void **)&rxr->rx_buf_ring; ring->fw_ring_id = INVALID_HW_RING_ID; @@ -1136,6 +1141,12 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) prod = RING_NEXT(rxr->rx_ring_struct, prod); } + /* Initialize dummy mbuf pointers for vector mode rx. */ + for (i = ring->ring_size; + i < ring->ring_size + RTE_BNXT_DESCS_PER_LOOP; i++) { + rxr->rx_buf_ring[i] = &rxq->fake_mbuf; + } + ring = rxr->ag_ring_struct; type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG; bnxt_init_rxbds(ring, type, size); diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_common.h b/drivers/net/bnxt/bnxt_rxtx_vec_common.h index 819b8290e4..8c10fdfa10 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_common.h +++ b/drivers/net/bnxt/bnxt_rxtx_vec_common.h @@ -75,6 +75,8 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) if (rte_mempool_get_bulk(rxq->mb_pool, (void *)rx_bufs, nb) < 0) { rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += nb; + for (i = 0; i < nb; i++) + rx_bufs[i] = &rxq->fake_mbuf; return; } diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index 24f9fc3c39..e7fe9325ab 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -22,52 +22,151 @@ * RX Ring handling */ -static uint32_t -bnxt_parse_pkt_type(uint32x4_t mm_rxcmp, uint32x4_t mm_rxcmp1) -{ - uint32_t flags_type, flags2; - uint8_t index; - - flags_type = vgetq_lane_u32(mm_rxcmp, 0); - flags2 = (uint16_t)vgetq_lane_u32(mm_rxcmp1, 0); - - /* - * Index format: - * bit 0: RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC - * bit 1: RX_CMPL_FLAGS2_IP_TYPE - * bit 2: RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN - * bits 3-6: RX_PKT_CMPL_FLAGS_ITYPE - */ - index = ((flags_type & RX_PKT_CMPL_FLAGS_ITYPE_MASK) >> 9) | - ((flags2 & (RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)) >> 2) | - ((flags2 & RX_PKT_CMPL_FLAGS2_IP_TYPE) >> 7); +#define GET_OL_FLAGS(rss_flags, ol_idx, errors, pi, ol_flags) \ +{ \ + uint32_t tmp, of; \ + \ + of = vgetq_lane_u32((rss_flags), (pi)) | \ + bnxt_ol_flags_table[vgetq_lane_u32((ol_idx), (pi))]; \ + \ + tmp = vgetq_lane_u32((errors), (pi)); \ + if (tmp) \ + of |= bnxt_ol_flags_err_table[tmp]; \ + (ol_flags) = of; \ +} - return bnxt_ptype_table[index]; +#define GET_DESC_FIELDS(rxcmp, rxcmp1, shuf_msk, ptype_idx, pkt_idx, ret) \ +{ \ + uint32_t ptype; \ + uint16_t vlan_tci; \ + uint32x4_t r; \ + \ + /* Set mbuf pkt_len, data_len, and rss_hash fields. */ \ + r = vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(rxcmp), \ + (shuf_msk))); \ + \ + /* Set packet type. */ \ + ptype = bnxt_ptype_table[vgetq_lane_u32((ptype_idx), (pkt_idx))]; \ + r = vsetq_lane_u32(ptype, r, 0); \ + \ + /* Set vlan_tci. */ \ + vlan_tci = vgetq_lane_u32((rxcmp1), 1); \ + r = vreinterpretq_u32_u16(vsetq_lane_u16(vlan_tci, \ + vreinterpretq_u16_u32(r), 5)); \ + (ret) = r; \ } -static uint32_t -bnxt_set_ol_flags(uint32x4_t mm_rxcmp, uint32x4_t mm_rxcmp1) +static void +descs_to_mbufs(uint32x4_t mm_rxcmp[4], uint32x4_t mm_rxcmp1[4], + uint64x2_t mb_init, struct rte_mbuf **mbuf) { - uint16_t flags_type, errors, flags; + const uint8x16_t shuf_msk = { + 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type (zeroes) */ + 2, 3, 0xFF, 0xFF, /* pkt_len */ + 2, 3, /* data_len */ + 0xFF, 0xFF, /* vlan_tci (zeroes) */ + 12, 13, 14, 15 /* rss hash */ + }; + const uint32x4_t flags_type_mask = { + RX_PKT_CMPL_FLAGS_ITYPE_MASK, + RX_PKT_CMPL_FLAGS_ITYPE_MASK, + RX_PKT_CMPL_FLAGS_ITYPE_MASK, + RX_PKT_CMPL_FLAGS_ITYPE_MASK + }; + const uint32x4_t flags2_mask1 = { + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC + }; + const uint32x4_t flags2_mask2 = { + RX_PKT_CMPL_FLAGS2_IP_TYPE, + RX_PKT_CMPL_FLAGS2_IP_TYPE, + RX_PKT_CMPL_FLAGS2_IP_TYPE, + RX_PKT_CMPL_FLAGS2_IP_TYPE + }; + const uint32x4_t rss_mask = { + RX_PKT_CMPL_FLAGS_RSS_VALID, + RX_PKT_CMPL_FLAGS_RSS_VALID, + RX_PKT_CMPL_FLAGS_RSS_VALID, + RX_PKT_CMPL_FLAGS_RSS_VALID + }; + const uint32x4_t flags2_index_mask = { + 0x1F, 0x1F, 0x1F, 0x1F + }; + const uint32x4_t flags2_error_mask = { + 0xF, 0xF, 0xF, 0xF + }; + uint32x4_t flags_type, flags2, index, errors, rss_flags; + uint32x4_t tmp, ptype_idx; + uint64x2_t t0, t1; uint32_t ol_flags; - /* Extract rxcmp1->flags2. */ - flags = vgetq_lane_u32(mm_rxcmp1, 0) & 0x1F; - /* Extract rxcmp->flags_type. */ - flags_type = vgetq_lane_u32(mm_rxcmp, 0); - /* Extract rxcmp1->errors_v2. */ - errors = (vgetq_lane_u32(mm_rxcmp1, 2) >> 4) & flags & 0xF; - - ol_flags = bnxt_ol_flags_table[flags & ~errors]; - - if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - ol_flags |= PKT_RX_RSS_HASH; - - if (errors) - ol_flags |= bnxt_ol_flags_err_table[errors]; - - return ol_flags; + /* Compute packet type table indexes for four packets */ + t0 = vreinterpretq_u64_u32(vzip1q_u32(mm_rxcmp[0], mm_rxcmp[1])); + t1 = vreinterpretq_u64_u32(vzip1q_u32(mm_rxcmp[2], mm_rxcmp[3])); + + flags_type = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(t0), + vget_low_u64(t1))); + ptype_idx = + vshrq_n_u32(vandq_u32(flags_type, flags_type_mask), 9); + + t0 = vreinterpretq_u64_u32(vzip1q_u32(mm_rxcmp1[0], mm_rxcmp1[1])); + t1 = vreinterpretq_u64_u32(vzip1q_u32(mm_rxcmp1[2], mm_rxcmp1[3])); + + flags2 = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(t0), + vget_low_u64(t1))); + + ptype_idx = vorrq_u32(ptype_idx, + vshrq_n_u32(vandq_u32(flags2, flags2_mask1), 2)); + ptype_idx = vorrq_u32(ptype_idx, + vshrq_n_u32(vandq_u32(flags2, flags2_mask2), 7)); + + /* Extract RSS valid flags for four packets. */ + rss_flags = vshrq_n_u32(vandq_u32(flags_type, rss_mask), 9); + + flags2 = vandq_u32(flags2, flags2_index_mask); + + /* Extract errors_v2 fields for four packets. */ + t0 = vreinterpretq_u64_u32(vzip2q_u32(mm_rxcmp1[0], mm_rxcmp1[1])); + t1 = vreinterpretq_u64_u32(vzip2q_u32(mm_rxcmp1[2], mm_rxcmp1[3])); + + errors = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(t0), + vget_low_u64(t1))); + + /* Compute ol_flags and checksum error indexes for four packets. */ + errors = vandq_u32(vshrq_n_u32(errors, 4), flags2_error_mask); + errors = vandq_u32(errors, flags2); + + index = vbicq_u32(flags2, errors); + + /* Update mbuf rearm_data for four packets. */ + GET_OL_FLAGS(rss_flags, index, errors, 0, ol_flags); + vst1q_u32((uint32_t *)&mbuf[0]->rearm_data, + vsetq_lane_u32(ol_flags, vreinterpretq_u32_u64(mb_init), 2)); + GET_OL_FLAGS(rss_flags, index, errors, 1, ol_flags); + vst1q_u32((uint32_t *)&mbuf[1]->rearm_data, + vsetq_lane_u32(ol_flags, vreinterpretq_u32_u64(mb_init), 2)); + GET_OL_FLAGS(rss_flags, index, errors, 2, ol_flags); + vst1q_u32((uint32_t *)&mbuf[2]->rearm_data, + vsetq_lane_u32(ol_flags, vreinterpretq_u32_u64(mb_init), 2)); + GET_OL_FLAGS(rss_flags, index, errors, 3, ol_flags); + vst1q_u32((uint32_t *)&mbuf[3]->rearm_data, + vsetq_lane_u32(ol_flags, vreinterpretq_u32_u64(mb_init), 2)); + + /* Update mbuf rx_descriptor_fields1 for four packets. */ + GET_DESC_FIELDS(mm_rxcmp[0], mm_rxcmp1[0], shuf_msk, ptype_idx, 0, tmp); + vst1q_u32((uint32_t *)&mbuf[0]->rx_descriptor_fields1, tmp); + GET_DESC_FIELDS(mm_rxcmp[1], mm_rxcmp1[1], shuf_msk, ptype_idx, 1, tmp); + vst1q_u32((uint32_t *)&mbuf[1]->rx_descriptor_fields1, tmp); + GET_DESC_FIELDS(mm_rxcmp[2], mm_rxcmp1[2], shuf_msk, ptype_idx, 2, tmp); + vst1q_u32((uint32_t *)&mbuf[2]->rx_descriptor_fields1, tmp); + GET_DESC_FIELDS(mm_rxcmp[3], mm_rxcmp1[3], shuf_msk, ptype_idx, 3, tmp); + vst1q_u32((uint32_t *)&mbuf[3]->rx_descriptor_fields1, tmp); } uint16_t @@ -77,17 +176,23 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, struct bnxt_rx_queue *rxq = rx_queue; struct bnxt_cp_ring_info *cpr = rxq->cp_ring; struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + uint16_t cp_ring_size = cpr->cp_ring_struct->ring_size; + uint16_t rx_ring_size = rxr->rx_ring_struct->ring_size; + struct cmpl_base *cp_desc_ring = cpr->cp_desc_ring; + uint64_t valid, desc_valid_mask = ~0UL; + const uint32x4_t info3_v_mask = { + CMPL_BASE_V, CMPL_BASE_V, + CMPL_BASE_V, CMPL_BASE_V + }; uint32_t raw_cons = cpr->cp_raw_cons; - uint32_t cons; + uint32_t cons, mbcons; int nb_rx_pkts = 0; - struct rx_pkt_cmpl *rxcmp; - const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0}; - const uint8x16_t shuf_msk = { - 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type (zeroes) */ - 2, 3, 0xFF, 0xFF, /* pkt_len */ - 2, 3, /* data_len */ - 0xFF, 0xFF, /* vlan_tci (zeroes) */ - 12, 13, 14, 15 /* rss hash */ + const uint64x2_t mb_init = {rxq->mbuf_initializer, 0}; + const uint32x4_t valid_target = { + !!(raw_cons & cp_ring_size), + !!(raw_cons & cp_ring_size), + !!(raw_cons & cp_ring_size), + !!(raw_cons & cp_ring_size) }; int i; @@ -101,72 +206,130 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, /* Return no more than RTE_BNXT_MAX_RX_BURST per call. */ nb_pkts = RTE_MIN(nb_pkts, RTE_BNXT_MAX_RX_BURST); - /* Make nb_pkts an integer multiple of RTE_BNXT_DESCS_PER_LOOP. */ - nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_BNXT_DESCS_PER_LOOP); - if (!nb_pkts) - return 0; + cons = raw_cons & (cp_ring_size - 1); + mbcons = (raw_cons / 2) & (rx_ring_size - 1); - /* Handle RX burst request */ - for (i = 0; i < nb_pkts; i++) { - uint32x4_t mm_rxcmp, mm_rxcmp1; - struct rx_pkt_cmpl_hi *rxcmp1; - uint32x4_t pkt_mb, rearm; - uint32_t ptype, ol_flags; - struct rte_mbuf *mbuf; - uint16_t vlan_tci; - uint16x8_t tmp16; - uint8x16_t tmp; + /* Prefetch first four descriptor pairs. */ + rte_prefetch0(&cp_desc_ring[cons]); + rte_prefetch0(&cp_desc_ring[cons + 4]); - cons = RING_CMP(cpr->cp_ring_struct, raw_cons); + /* Ensure that we do not go past the ends of the rings. */ + nb_pkts = RTE_MIN(nb_pkts, RTE_MIN(rx_ring_size - mbcons, + (cp_ring_size - cons) / 2)); + /* + * If we are at the end of the ring, ensure that descriptors after the + * last valid entry are not treated as valid. Otherwise, force the + * maximum number of packets to receive to be a multiple of the per- + * loop count. + */ + if (nb_pkts < RTE_BNXT_DESCS_PER_LOOP) + desc_valid_mask >>= 16 * (RTE_BNXT_DESCS_PER_LOOP - nb_pkts); + else + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_BNXT_DESCS_PER_LOOP); - rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; - rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cons + 1]; + /* Handle RX burst request */ + for (i = 0; i < nb_pkts; i += RTE_BNXT_DESCS_PER_LOOP, + cons += RTE_BNXT_DESCS_PER_LOOP * 2, + mbcons += RTE_BNXT_DESCS_PER_LOOP) { + uint32x4_t rxcmp1[RTE_BNXT_DESCS_PER_LOOP]; + uint32x4_t rxcmp[RTE_BNXT_DESCS_PER_LOOP]; + uint32x4_t info3_v; + uint64x2_t t0, t1; + uint32_t num_valid; + + /* Copy four mbuf pointers to output array. */ + t0 = vld1q_u64((void *)&rxr->rx_buf_ring[mbcons]); +#ifdef RTE_ARCH_ARM64 + t1 = vld1q_u64((void *)&rxr->rx_buf_ring[mbcons + 2]); +#endif + vst1q_u64((void *)&rx_pkts[i], t0); +#ifdef RTE_ARCH_ARM64 + vst1q_u64((void *)&rx_pkts[i + 2], t1); +#endif + + /* Prefetch four descriptor pairs for next iteration. */ + if (i + RTE_BNXT_DESCS_PER_LOOP < nb_pkts) { + rte_prefetch0(&cp_desc_ring[cons + 8]); + rte_prefetch0(&cp_desc_ring[cons + 12]); + } - if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) + /* + * Load the four curent descriptors into SSE registers in + * reverse order to ensure consistent state. + */ + rxcmp1[3] = vld1q_u32((void *)&cpr->cp_desc_ring[cons + 7]); + rte_cio_rmb(); + rxcmp[3] = vld1q_u32((void *)&cpr->cp_desc_ring[cons + 6]); + + rxcmp1[2] = vld1q_u32((void *)&cpr->cp_desc_ring[cons + 5]); + rte_cio_rmb(); + rxcmp[2] = vld1q_u32((void *)&cpr->cp_desc_ring[cons + 4]); + + t1 = vreinterpretq_u64_u32(vzip2q_u32(rxcmp1[2], rxcmp1[3])); + + rxcmp1[1] = vld1q_u32((void *)&cpr->cp_desc_ring[cons + 3]); + rte_cio_rmb(); + rxcmp[1] = vld1q_u32((void *)&cpr->cp_desc_ring[cons + 2]); + + rxcmp1[0] = vld1q_u32((void *)&cpr->cp_desc_ring[cons + 1]); + rte_cio_rmb(); + rxcmp[0] = vld1q_u32((void *)&cpr->cp_desc_ring[cons + 0]); + + t0 = vreinterpretq_u64_u32(vzip2q_u32(rxcmp1[0], rxcmp1[1])); + + /* Isolate descriptor status flags. */ + info3_v = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(t0), + vget_low_u64(t1))); + info3_v = vandq_u32(info3_v, info3_v_mask); + info3_v = veorq_u32(info3_v, valid_target); + + /* + * Pack the 128-bit array of valid descriptor flags into 64 + * bits and count the number of set bits in order to determine + * the number of valid descriptors. + */ + valid = vget_lane_u64(vreinterpret_u64_u16(vqmovn_u32(info3_v)), + 0); + /* + * At this point, 'valid' is a 64-bit value containing four + * 16-bit fields, each of which is either 0x0001 or 0x0000. + * Compute number of valid descriptors from the index of + * the highest non-zero field. + */ + num_valid = (sizeof(uint64_t) / sizeof(uint16_t)) - + (__builtin_clzl(valid & desc_valid_mask) / 16); + + switch (num_valid) { + case 4: + rxr->rx_buf_ring[mbcons + 3] = NULL; + /* FALLTHROUGH */ + case 3: + rxr->rx_buf_ring[mbcons + 2] = NULL; + /* FALLTHROUGH */ + case 2: + rxr->rx_buf_ring[mbcons + 1] = NULL; + /* FALLTHROUGH */ + case 1: + rxr->rx_buf_ring[mbcons + 0] = NULL; break; + case 0: + goto out; + } - mm_rxcmp = vld1q_u32((uint32_t *)rxcmp); - mm_rxcmp1 = vld1q_u32((uint32_t *)rxcmp); - raw_cons += 2; - cons = rxcmp->opaque; - - mbuf = rxr->rx_buf_ring[cons]; - rte_prefetch0(mbuf); - rxr->rx_buf_ring[cons] = NULL; - - /* Set fields from mbuf initializer and ol_flags. */ - ol_flags = bnxt_set_ol_flags(mm_rxcmp, mm_rxcmp1); - rearm = vsetq_lane_u32(ol_flags, - vreinterpretq_u32_u64(mbuf_init), 2); - vst1q_u32((uint32_t *)&mbuf->rearm_data, rearm); - - /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - tmp = vqtbl1q_u8(vreinterpretq_u8_u32(mm_rxcmp), shuf_msk); - pkt_mb = vreinterpretq_u32_u8(tmp); - - /* Set packet type. */ - ptype = bnxt_parse_pkt_type(mm_rxcmp, mm_rxcmp1); - pkt_mb = vsetq_lane_u32(ptype, pkt_mb, 0); - - /* Set vlan_tci. */ - vlan_tci = vgetq_lane_u32(mm_rxcmp1, 1); - tmp16 = vsetq_lane_u16(vlan_tci, - vreinterpretq_u16_u32(pkt_mb), - 5); - pkt_mb = vreinterpretq_u32_u16(tmp16); - - /* Store descriptor fields. */ - vst1q_u32((uint32_t *)&mbuf->rx_descriptor_fields1, pkt_mb); + descs_to_mbufs(rxcmp, rxcmp1, mb_init, &rx_pkts[nb_rx_pkts]); + nb_rx_pkts += num_valid; - rx_pkts[nb_rx_pkts++] = mbuf; + if (num_valid < RTE_BNXT_DESCS_PER_LOOP) + break; } +out: if (nb_rx_pkts) { rxr->rx_prod = RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); rxq->rxrearm_nb += nb_rx_pkts; - cpr->cp_raw_cons = raw_cons; + cpr->cp_raw_cons += 2 * nb_rx_pkts; cpr->valid = !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); bnxt_db_cq(cpr); diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index 7e87555408..362992ceb2 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -1,5 +1,5 @@ -// SPDX-License-Identifier: BSD-3-Clause -/* Copyright(c) 2019 Broadcom All rights reserved. */ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2019-2020 Broadcom All rights reserved. */ #include #include @@ -8,11 +8,7 @@ #include #include #include -#if defined(RTE_ARCH_X86) -#include -#else -#error "bnxt vector pmd: unsupported target." -#endif +#include #include "bnxt.h" #include "bnxt_cpr.h" @@ -26,52 +22,135 @@ * RX Ring handling */ -static __m128i -bnxt_parse_pkt_type(__m128i mm_rxcmp, __m128i mm_rxcmp1) -{ - uint32_t flags_type, flags2; - uint8_t index; - - flags_type = _mm_extract_epi16(mm_rxcmp, 0); - flags2 = _mm_extract_epi32(mm_rxcmp1, 0); - - /* - * Index format: - * bit 0: RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC - * bit 1: RX_CMPL_FLAGS2_IP_TYPE - * bit 2: RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN - * bits 3-6: RX_PKT_CMPL_FLAGS_ITYPE - */ - index = ((flags_type & RX_PKT_CMPL_FLAGS_ITYPE_MASK) >> 9) | - ((flags2 & (RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)) >> 2) | - ((flags2 & RX_PKT_CMPL_FLAGS2_IP_TYPE) >> 7); +#define GET_OL_FLAGS(rss_flags, ol_index, errors, pi, ol_flags) \ +{ \ + uint32_t tmp, of; \ + \ + of = _mm_extract_epi32((rss_flags), (pi)) | \ + bnxt_ol_flags_table[_mm_extract_epi32((ol_index), (pi))]; \ + \ + tmp = _mm_extract_epi32((errors), (pi)); \ + if (tmp) \ + of |= bnxt_ol_flags_err_table[tmp]; \ + (ol_flags) = of; \ +} - return _mm_set_epi32(0, 0, 0, bnxt_ptype_table[index]); +#define GET_DESC_FIELDS(rxcmp, rxcmp1, shuf_msk, ptype_idx, pi, ret) \ +{ \ + uint32_t ptype; \ + __m128i r; \ + \ + /* Set mbuf pkt_len, data_len, and rss_hash fields. */ \ + r = _mm_shuffle_epi8((rxcmp), (shuf_msk)); \ + \ + /* Set packet type. */ \ + ptype = bnxt_ptype_table[_mm_extract_epi32((ptype_idx), (pi))]; \ + r = _mm_blend_epi16(r, _mm_set_epi32(0, 0, 0, ptype), 0x3); \ + \ + /* Set vlan_tci. */ \ + r = _mm_blend_epi16(r, _mm_slli_si128((rxcmp1), 6), 0x20); \ + (ret) = r; \ } -static __m128i -bnxt_set_ol_flags(__m128i mm_rxcmp, __m128i mm_rxcmp1) +static inline void +descs_to_mbufs(__m128i mm_rxcmp[4], __m128i mm_rxcmp1[4], + __m128i mbuf_init, struct rte_mbuf **mbuf) { - uint16_t flags_type, errors, flags; + const __m128i shuf_msk = + _mm_set_epi8(15, 14, 13, 12, /* rss */ + 0xFF, 0xFF, /* vlan_tci (zeroes) */ + 3, 2, /* data_len */ + 0xFF, 0xFF, 3, 2, /* pkt_len */ + 0xFF, 0xFF, 0xFF, 0xFF); /* pkt_type (zeroes) */ + const __m128i flags_type_mask = + _mm_set_epi32(RX_PKT_CMPL_FLAGS_ITYPE_MASK, + RX_PKT_CMPL_FLAGS_ITYPE_MASK, + RX_PKT_CMPL_FLAGS_ITYPE_MASK, + RX_PKT_CMPL_FLAGS_ITYPE_MASK); + const __m128i flags2_mask1 = + _mm_set_epi32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC); + const __m128i flags2_mask2 = + _mm_set_epi32(RX_PKT_CMPL_FLAGS2_IP_TYPE, + RX_PKT_CMPL_FLAGS2_IP_TYPE, + RX_PKT_CMPL_FLAGS2_IP_TYPE, + RX_PKT_CMPL_FLAGS2_IP_TYPE); + const __m128i rss_mask = + _mm_set_epi32(RX_PKT_CMPL_FLAGS_RSS_VALID, + RX_PKT_CMPL_FLAGS_RSS_VALID, + RX_PKT_CMPL_FLAGS_RSS_VALID, + RX_PKT_CMPL_FLAGS_RSS_VALID); + __m128i t0, t1, flags_type, flags2, index, errors, rss_flags; + __m128i ptype_idx; uint32_t ol_flags; - /* Extract rxcmp1->flags2. */ - flags = _mm_extract_epi32(mm_rxcmp1, 0) & 0x1F; - /* Extract rxcmp->flags_type. */ - flags_type = _mm_extract_epi16(mm_rxcmp, 0); - /* Extract rxcmp1->errors_v2. */ - errors = (_mm_extract_epi16(mm_rxcmp1, 4) >> 4) & flags & 0xF; + /* Compute packet type table indexes for four packets */ + t0 = _mm_unpacklo_epi32(mm_rxcmp[0], mm_rxcmp[1]); + t1 = _mm_unpacklo_epi32(mm_rxcmp[2], mm_rxcmp[3]); + flags_type = _mm_unpacklo_epi64(t0, t1); + ptype_idx = + _mm_srli_epi32(_mm_and_si128(flags_type, flags_type_mask), 9); - ol_flags = bnxt_ol_flags_table[flags & ~errors]; + t0 = _mm_unpacklo_epi32(mm_rxcmp1[0], mm_rxcmp1[1]); + t1 = _mm_unpacklo_epi32(mm_rxcmp1[2], mm_rxcmp1[3]); + flags2 = _mm_unpacklo_epi64(t0, t1); - if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - ol_flags |= PKT_RX_RSS_HASH; + ptype_idx = _mm_or_si128(ptype_idx, + _mm_srli_epi32(_mm_and_si128(flags2, flags2_mask1), 2)); + ptype_idx = _mm_or_si128(ptype_idx, + _mm_srli_epi32(_mm_and_si128(flags2, flags2_mask2), 7)); - if (errors) - ol_flags |= bnxt_ol_flags_err_table[errors]; + /* Extract RSS valid flags for four packets. */ + rss_flags = _mm_srli_epi32(_mm_and_si128(flags_type, rss_mask), 9); - return _mm_set_epi64x(ol_flags, 0); + /* Extract errors_v2 fields for four packets. */ + t0 = _mm_unpackhi_epi32(mm_rxcmp1[0], mm_rxcmp1[1]); + t1 = _mm_unpackhi_epi32(mm_rxcmp1[2], mm_rxcmp1[3]); + + /* Compute ol_flags and checksum error indexes for four packets. */ + flags2 = _mm_and_si128(flags2, _mm_set_epi32(0x1F, 0x1F, 0x1F, 0x1F)); + + errors = _mm_srli_epi32(_mm_unpacklo_epi64(t0, t1), 4); + errors = _mm_and_si128(errors, _mm_set_epi32(0xF, 0xF, 0xF, 0xF)); + errors = _mm_and_si128(errors, flags2); + + index = _mm_andnot_si128(errors, flags2); + + /* Update mbuf rearm_data for four packets. */ + GET_OL_FLAGS(rss_flags, index, errors, 0, ol_flags); + _mm_store_si128((void *)&mbuf[0]->rearm_data, + _mm_or_si128(mbuf_init, _mm_set_epi64x(ol_flags, 0))); + + GET_OL_FLAGS(rss_flags, index, errors, 1, ol_flags); + _mm_store_si128((void *)&mbuf[1]->rearm_data, + _mm_or_si128(mbuf_init, _mm_set_epi64x(ol_flags, 0))); + + GET_OL_FLAGS(rss_flags, index, errors, 2, ol_flags); + _mm_store_si128((void *)&mbuf[2]->rearm_data, + _mm_or_si128(mbuf_init, _mm_set_epi64x(ol_flags, 0))); + + GET_OL_FLAGS(rss_flags, index, errors, 3, ol_flags); + _mm_store_si128((void *)&mbuf[3]->rearm_data, + _mm_or_si128(mbuf_init, _mm_set_epi64x(ol_flags, 0))); + + /* Update mbuf rx_descriptor_fields1 for four packes. */ + GET_DESC_FIELDS(mm_rxcmp[0], mm_rxcmp1[0], shuf_msk, ptype_idx, 0, t0); + _mm_store_si128((void *)&mbuf[0]->rx_descriptor_fields1, t0); + + GET_DESC_FIELDS(mm_rxcmp[1], mm_rxcmp1[1], shuf_msk, ptype_idx, 1, t0); + _mm_store_si128((void *)&mbuf[1]->rx_descriptor_fields1, t0); + + GET_DESC_FIELDS(mm_rxcmp[2], mm_rxcmp1[2], shuf_msk, ptype_idx, 2, t0); + _mm_store_si128((void *)&mbuf[2]->rx_descriptor_fields1, t0); + + GET_DESC_FIELDS(mm_rxcmp[3], mm_rxcmp1[3], shuf_msk, ptype_idx, 3, t0); + _mm_store_si128((void *)&mbuf[3]->rx_descriptor_fields1, t0); } uint16_t @@ -79,19 +158,23 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct bnxt_rx_queue *rxq = rx_queue; + const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); struct bnxt_cp_ring_info *cpr = rxq->cp_ring; struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + uint16_t cp_ring_size = cpr->cp_ring_struct->ring_size; + uint16_t rx_ring_size = rxr->rx_ring_struct->ring_size; + struct cmpl_base *cp_desc_ring = cpr->cp_desc_ring; + uint64_t valid, desc_valid_mask = ~0UL; + const __m128i info3_v_mask = _mm_set_epi32(CMPL_BASE_V, CMPL_BASE_V, + CMPL_BASE_V, CMPL_BASE_V); uint32_t raw_cons = cpr->cp_raw_cons; - uint32_t cons; + uint32_t cons, mbcons; int nb_rx_pkts = 0; - struct rx_pkt_cmpl *rxcmp; - const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); - const __m128i shuf_msk = - _mm_set_epi8(15, 14, 13, 12, /* rss */ - 0xFF, 0xFF, /* vlan_tci (zeroes) */ - 3, 2, /* data_len */ - 0xFF, 0xFF, 3, 2, /* pkt_len */ - 0xFF, 0xFF, 0xFF, 0xFF); /* pkt_type (zeroes) */ + const __m128i valid_target = + _mm_set_epi32(!!(raw_cons & cp_ring_size), + !!(raw_cons & cp_ring_size), + !!(raw_cons & cp_ring_size), + !!(raw_cons & cp_ring_size)); int i; /* If Rx Q was stopped return */ @@ -104,69 +187,120 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, /* Return no more than RTE_BNXT_MAX_RX_BURST per call. */ nb_pkts = RTE_MIN(nb_pkts, RTE_BNXT_MAX_RX_BURST); + cons = raw_cons & (cp_ring_size - 1); + mbcons = (raw_cons / 2) & (rx_ring_size - 1); + + /* Prefetch first four descriptor pairs. */ + rte_prefetch0(&cp_desc_ring[cons]); + rte_prefetch0(&cp_desc_ring[cons + 4]); + + /* Ensure that we do not go past the ends of the rings. */ + nb_pkts = RTE_MIN(nb_pkts, RTE_MIN(rx_ring_size - mbcons, + (cp_ring_size - cons) / 2)); /* - * Make nb_pkts an integer multiple of RTE_BNXT_DESCS_PER_LOOP. - * nb_pkts < RTE_BNXT_DESCS_PER_LOOP, just return no packet + * If we are at the end of the ring, ensure that descriptors after the + * last valid entry are not treated as valid. Otherwise, force the + * maximum number of packets to receive to be a multiple of the per- + * loop count. */ - nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_BNXT_DESCS_PER_LOOP); - if (!nb_pkts) - return 0; + if (nb_pkts < RTE_BNXT_DESCS_PER_LOOP) + desc_valid_mask >>= 16 * (RTE_BNXT_DESCS_PER_LOOP - nb_pkts); + else + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_BNXT_DESCS_PER_LOOP); /* Handle RX burst request */ - for (i = 0; i < nb_pkts; i++) { - struct rx_pkt_cmpl_hi *rxcmp1; - struct rte_mbuf *mbuf; - __m128i mm_rxcmp, mm_rxcmp1, pkt_mb, ptype, rearm; - - cons = RING_CMP(cpr->cp_ring_struct, raw_cons); + for (i = 0; i < nb_pkts; i += RTE_BNXT_DESCS_PER_LOOP, + cons += RTE_BNXT_DESCS_PER_LOOP * 2, + mbcons += RTE_BNXT_DESCS_PER_LOOP) { + __m128i rxcmp1[RTE_BNXT_DESCS_PER_LOOP]; + __m128i rxcmp[RTE_BNXT_DESCS_PER_LOOP]; + __m128i tmp0, tmp1, info3_v; + uint32_t num_valid; + + /* Copy four mbuf pointers to output array. */ + tmp0 = _mm_loadu_si128((void *)&rxr->rx_buf_ring[mbcons]); +#ifdef RTE_ARCH_X86_64 + tmp1 = _mm_loadu_si128((void *)&rxr->rx_buf_ring[mbcons + 2]); +#endif + _mm_storeu_si128((void *)&rx_pkts[i], tmp0); +#ifdef RTE_ARCH_X86_64 + _mm_storeu_si128((void *)&rx_pkts[i + 2], tmp1); +#endif - rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; - rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cons + 1]; + /* Prefetch four descriptor pairs for next iteration. */ + if (i + RTE_BNXT_DESCS_PER_LOOP < nb_pkts) { + rte_prefetch0(&cp_desc_ring[cons + 8]); + rte_prefetch0(&cp_desc_ring[cons + 12]); + } - if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) - break; + /* + * Load the four curent descriptors into SSE registers in + * reverse order to ensure consistent state. + */ + rxcmp1[3] = _mm_load_si128((void *)&cp_desc_ring[cons + 7]); + rte_compiler_barrier(); + rxcmp[3] = _mm_load_si128((void *)&cp_desc_ring[cons + 6]); - mm_rxcmp = _mm_load_si128((__m128i *)rxcmp); - mm_rxcmp1 = _mm_load_si128((__m128i *)rxcmp1); + rxcmp1[2] = _mm_load_si128((void *)&cp_desc_ring[cons + 5]); + rte_compiler_barrier(); + rxcmp[2] = _mm_load_si128((void *)&cp_desc_ring[cons + 4]); - raw_cons += 2; - cons = rxcmp->opaque; + tmp1 = _mm_unpackhi_epi32(rxcmp1[2], rxcmp1[3]); - mbuf = rxr->rx_buf_ring[cons]; - rxr->rx_buf_ring[cons] = NULL; + rxcmp1[1] = _mm_load_si128((void *)&cp_desc_ring[cons + 3]); + rte_compiler_barrier(); + rxcmp[1] = _mm_load_si128((void *)&cp_desc_ring[cons + 2]); - /* Set fields from mbuf initializer and ol_flags. */ - rearm = _mm_or_si128(mbuf_init, - bnxt_set_ol_flags(mm_rxcmp, mm_rxcmp1)); - _mm_store_si128((__m128i *)&mbuf->rearm_data, rearm); + rxcmp1[0] = _mm_load_si128((void *)&cp_desc_ring[cons + 1]); + rte_compiler_barrier(); + rxcmp[0] = _mm_load_si128((void *)&cp_desc_ring[cons + 0]); - /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - pkt_mb = _mm_shuffle_epi8(mm_rxcmp, shuf_msk); + tmp0 = _mm_unpackhi_epi32(rxcmp1[0], rxcmp1[1]); - /* Set packet type. */ - ptype = bnxt_parse_pkt_type(mm_rxcmp, mm_rxcmp1); - pkt_mb = _mm_blend_epi16(pkt_mb, ptype, 0x3); + /* Isolate descriptor valid flags. */ + info3_v = _mm_and_si128(_mm_unpacklo_epi64(tmp0, tmp1), + info3_v_mask); + info3_v = _mm_xor_si128(info3_v, valid_target); /* - * Shift vlan_tci from completion metadata field left six - * bytes and blend into mbuf->rx_descriptor_fields1 to set - * mbuf->vlan_tci. + * Pack the 128-bit array of valid descriptor flags into 64 + * bits and count the number of set bits in order to determine + * the number of valid descriptors. */ - pkt_mb = _mm_blend_epi16(pkt_mb, - _mm_slli_si128(mm_rxcmp1, 6), 0x20); + valid = _mm_cvtsi128_si64(_mm_packs_epi32(info3_v, info3_v)); + num_valid = __builtin_popcountll(valid & desc_valid_mask); + + switch (num_valid) { + case 4: + rxr->rx_buf_ring[mbcons + 3] = NULL; + /* FALLTHROUGH */ + case 3: + rxr->rx_buf_ring[mbcons + 2] = NULL; + /* FALLTHROUGH */ + case 2: + rxr->rx_buf_ring[mbcons + 1] = NULL; + /* FALLTHROUGH */ + case 1: + rxr->rx_buf_ring[mbcons + 0] = NULL; + break; + case 0: + goto out; + } - /* Store descriptor fields. */ - _mm_storeu_si128((void *)&mbuf->rx_descriptor_fields1, pkt_mb); + descs_to_mbufs(rxcmp, rxcmp1, mbuf_init, &rx_pkts[nb_rx_pkts]); + nb_rx_pkts += num_valid; - rx_pkts[nb_rx_pkts++] = mbuf; + if (num_valid < RTE_BNXT_DESCS_PER_LOOP) + break; } +out: if (nb_rx_pkts) { rxr->rx_prod = RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); rxq->rxrearm_nb += nb_rx_pkts; - cpr->cp_raw_cons = raw_cons; + cpr->cp_raw_cons += 2 * nb_rx_pkts; cpr->valid = !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); bnxt_db_cq(cpr); From patchwork Wed Sep 9 15:57:30 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lance Richardson X-Patchwork-Id: 77076 X-Patchwork-Delegate: ajit.khaparde@broadcom.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6A737A04B5; Wed, 9 Sep 2020 17:57:38 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 498751C115; Wed, 9 Sep 2020 17:57:38 +0200 (CEST) Received: from mail-pf1-f195.google.com (mail-pf1-f195.google.com [209.85.210.195]) by dpdk.org (Postfix) with ESMTP id 0BA721C112 for ; Wed, 9 Sep 2020 17:57:36 +0200 (CEST) Received: by mail-pf1-f195.google.com with SMTP id w7so2541467pfi.4 for ; Wed, 09 Sep 2020 08:57:35 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=broadcom.com; s=google; h=from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=hISRRUNg7+Niq+Fa+X0zHfPM0eE3nGBX4GiZaG41v9g=; b=JCG3HGns1JbDoBnbvPcAukK6Jyq08V2xHQmV1p9Gl46Qr5DG+vHdMMlcixNdWKrkV6 LQXA1ERatf5vNUI2CtwCoyUBVQzZqdPDex4gbvDyzFSLHnJxmUayO5qU5dGnQdGpMQBH 2aM3zSQRcr3ryN0Mg4oTwiAxdkGuwk4Uoe1lQ= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=hISRRUNg7+Niq+Fa+X0zHfPM0eE3nGBX4GiZaG41v9g=; b=q0+1282pkGhPTKETX0tQu/qXjpDtnScf6I/8mkb/bgnYj7mL7VF7d7iZ5J0ZXcVv2V qG12xVTgAvAndKbbjswtp7DwhbeJzbbzqx7vS4sLpK9YsO553qHz5jNwK/+Uvu8WN3e2 KxKaK6zLkzTSSk4+vcs0w/RABMuNfH+ASSfC3lizueC7IsRsYe1AjTotM3olyae2Ibbr fWtV64IZmj7fKDlHdIndnDch5wzOKct/0jWr6plj3j20OK3LETRNvrCmqifQ6tzALqkn UBORkMFyy41+b/KLCtZzalrYco/5K4CmfQcAEbX7jkB7iuhUfEq5CCWVt0dAY5SHSrXX q5dg== X-Gm-Message-State: AOAM530wapUkr508sXtlxhADLP1sUHjhiTPKrdpRdRLbjBvNkn35pshx 4hqN6y5RcIEsdeLpjI+e8g9VHA== X-Google-Smtp-Source: ABdhPJzip5BsKG6K9FjKTjrvxzMyjDMn5VbqVeeXyPY+QeTeLhT5NvYjOCocmZdkQ/BsmqxmRcsJxQ== X-Received: by 2002:a17:902:aa8d:: with SMTP id d13mr1447191plr.124.1599667055212; Wed, 09 Sep 2020 08:57:35 -0700 (PDT) Received: from localhost.localdomain ([192.19.231.250]) by smtp.gmail.com with ESMTPSA id kt18sm77948pjb.56.2020.09.09.08.57.34 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 09 Sep 2020 08:57:34 -0700 (PDT) From: Lance Richardson To: Ajit Khaparde , Somnath Kotur Cc: dev@dpdk.org Date: Wed, 9 Sep 2020 11:57:30 -0400 Message-Id: <20200909155730.29182-1-lance.richardson@broadcom.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 12/12] net/bnxt: transmit vector mode improvements X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Improve performance of vector burst transmit function by processing multiple packets per inner loop iteration. Signed-off-by: Lance Richardson --- drivers/net/bnxt/bnxt_rxtx_vec_sse.c | 96 +++++++++++++++++++--------- 1 file changed, 66 insertions(+), 30 deletions(-) diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index 362992ceb2..dace2f8a16 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -316,13 +316,14 @@ bnxt_tx_cmp_vec(struct bnxt_tx_queue *txq, int nr_pkts) struct rte_mbuf **free = txq->free; uint16_t cons = txr->tx_cons; unsigned int blk = 0; + uint32_t ring_mask = txr->tx_ring_struct->ring_mask; while (nr_pkts--) { struct bnxt_sw_tx_bd *tx_buf; struct rte_mbuf *mbuf; tx_buf = &txr->tx_buf_ring[cons]; - cons = RING_NEXT(txr->tx_ring_struct, cons); + cons = (cons + 1) & ring_mask; mbuf = rte_pktmbuf_prefree_seg(tx_buf->mbuf); if (unlikely(mbuf == NULL)) continue; @@ -376,18 +377,40 @@ bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq) } } +static inline void +bnxt_xmit_one(struct rte_mbuf *mbuf, struct tx_bd_long *txbd, + struct bnxt_sw_tx_bd *tx_buf) +{ + __m128i desc; + + tx_buf->mbuf = mbuf; + tx_buf->nr_bds = 1; + + desc = _mm_set_epi64x(mbuf->buf_iova + mbuf->data_off, + bnxt_xmit_flags_len(mbuf->data_len, + TX_BD_FLAGS_NOCMPL)); + desc = _mm_blend_epi16(desc, _mm_set_epi16(0, 0, 0, 0, 0, 0, + mbuf->data_len, 0), 0x02); + _mm_store_si128((void *)txbd, desc); +} + static uint16_t -bnxt_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, +bnxt_xmit_fixed_burst_vec(struct bnxt_tx_queue *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { - struct bnxt_tx_queue *txq = tx_queue; struct bnxt_tx_ring_info *txr = txq->tx_ring; - uint16_t prod = txr->tx_prod; - struct rte_mbuf *tx_mbuf; - struct tx_bd_long *txbd = NULL; + uint16_t tx_prod = txr->tx_prod; + struct tx_bd_long *txbd; struct bnxt_sw_tx_bd *tx_buf; uint16_t to_send; + txbd = &txr->tx_desc_ring[tx_prod]; + tx_buf = &txr->tx_buf_ring[tx_prod]; + + /* Prefetch next transmit buffer descriptors. */ + rte_prefetch0(txbd); + rte_prefetch0(txbd + 3); + nb_pkts = RTE_MIN(nb_pkts, bnxt_tx_avail(txq)); if (unlikely(nb_pkts == 0)) @@ -395,33 +418,35 @@ bnxt_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, /* Handle TX burst request */ to_send = nb_pkts; - while (to_send) { - tx_mbuf = *tx_pkts++; - rte_prefetch0(tx_mbuf); - - tx_buf = &txr->tx_buf_ring[prod]; - tx_buf->mbuf = tx_mbuf; - tx_buf->nr_bds = 1; - - txbd = &txr->tx_desc_ring[prod]; - txbd->address = tx_mbuf->buf_iova + tx_mbuf->data_off; - txbd->len = tx_mbuf->data_len; - txbd->flags_type = bnxt_xmit_flags_len(tx_mbuf->data_len, - TX_BD_FLAGS_NOCMPL); - prod = RING_NEXT(txr->tx_ring_struct, prod); - to_send--; + while (to_send >= RTE_BNXT_DESCS_PER_LOOP) { + /* Prefetch next transmit buffer descriptors. */ + rte_prefetch0(txbd + 4); + rte_prefetch0(txbd + 7); + + bnxt_xmit_one(tx_pkts[0], txbd++, tx_buf++); + bnxt_xmit_one(tx_pkts[1], txbd++, tx_buf++); + bnxt_xmit_one(tx_pkts[2], txbd++, tx_buf++); + bnxt_xmit_one(tx_pkts[3], txbd++, tx_buf++); + + to_send -= RTE_BNXT_DESCS_PER_LOOP; + tx_pkts += RTE_BNXT_DESCS_PER_LOOP; } - /* Request a completion for last packet in burst */ - if (txbd) { - txbd->opaque = nb_pkts; - txbd->flags_type &= ~TX_BD_LONG_FLAGS_NO_CMPL; + while (to_send) { + bnxt_xmit_one(tx_pkts[0], txbd++, tx_buf++); + to_send--; + tx_pkts++; } + /* Request a completion for the final packet of burst. */ rte_compiler_barrier(); - bnxt_db_write(&txr->tx_db, prod); + txbd[-1].opaque = nb_pkts; + txbd[-1].flags_type &= ~TX_BD_LONG_FLAGS_NO_CMPL; - txr->tx_prod = prod; + tx_prod = RING_ADV(txr->tx_ring_struct, tx_prod, nb_pkts); + bnxt_db_write(&txr->tx_db, tx_prod); + + txr->tx_prod = tx_prod; return nb_pkts; } @@ -432,6 +457,8 @@ bnxt_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, { int nb_sent = 0; struct bnxt_tx_queue *txq = tx_queue; + struct bnxt_tx_ring_info *txr = txq->tx_ring; + uint16_t ring_size = txr->tx_ring_struct->ring_size; /* Tx queue was stopped; wait for it to be restarted */ if (unlikely(!txq->tx_started)) { @@ -446,10 +473,19 @@ bnxt_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, while (nb_pkts) { uint16_t ret, num; + /* + * Ensure that no more than RTE_BNXT_MAX_TX_BURST packets + * are transmitted before the next completion. + */ num = RTE_MIN(nb_pkts, RTE_BNXT_MAX_TX_BURST); - ret = bnxt_xmit_fixed_burst_vec(tx_queue, - &tx_pkts[nb_sent], - num); + + /* + * Ensure that a ring wrap does not occur within a call to + * bnxt_xmit_fixed_burst_vec(). + */ + num = RTE_MIN(num, + ring_size - (txr->tx_prod & (ring_size - 1))); + ret = bnxt_xmit_fixed_burst_vec(txq, &tx_pkts[nb_sent], num); nb_sent += ret; nb_pkts -= ret; if (ret < num)