Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/135605/?format=api
{ "id": 135605, "url": "http://patches.dpdk.org/api/patches/135605/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20231227042119.72469-18-ajit.khaparde@broadcom.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20231227042119.72469-18-ajit.khaparde@broadcom.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20231227042119.72469-18-ajit.khaparde@broadcom.com", "date": "2023-12-27T04:21:18", "name": "[v3,17/18] net/bnxt: add AVX2 support for compressed CQE", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "5b3172762b2b43b415f16c739bbb200aad7e9e37", "submitter": { "id": 501, "url": "http://patches.dpdk.org/api/people/501/?format=api", "name": "Ajit Khaparde", "email": "ajit.khaparde@broadcom.com" }, "delegate": { "id": 1766, "url": "http://patches.dpdk.org/api/users/1766/?format=api", "username": "ajitkhaparde", "first_name": "Ajit", "last_name": "Khaparde", "email": "ajit.khaparde@broadcom.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20231227042119.72469-18-ajit.khaparde@broadcom.com/mbox/", "series": [ { "id": 30672, "url": "http://patches.dpdk.org/api/series/30672/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=30672", "date": "2023-12-27T04:21:01", "name": "bnxt patchset", "version": 3, "mbox": "http://patches.dpdk.org/series/30672/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/135605/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/135605/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 94CD9437A1;\n\tWed, 27 Dec 2023 05:23:43 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id A14DB40EA5;\n\tWed, 27 Dec 2023 05:21:54 +0100 (CET)", "from mail-qt1-f175.google.com (mail-qt1-f175.google.com\n [209.85.160.175])\n by mails.dpdk.org (Postfix) with ESMTP id 03B3B40E2D\n for <dev@dpdk.org>; Wed, 27 Dec 2023 05:21:50 +0100 (CET)", "by mail-qt1-f175.google.com with SMTP id\n d75a77b69052e-42786ec994bso44746361cf.1\n for <dev@dpdk.org>; Tue, 26 Dec 2023 20:21:49 -0800 (PST)", "from localhost.localdomain\n ([2605:a601:a780:1400:c066:75e3:74c8:50e6])\n by smtp.gmail.com with ESMTPSA id\n bt7-20020ac86907000000b00427e120889bsm1415488qtb.91.2023.12.26.20.21.47\n (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n Tue, 26 Dec 2023 20:21:48 -0800 (PST)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=broadcom.com; s=google; t=1703650909; x=1704255709; darn=dpdk.org;\n h=mime-version:references:in-reply-to:message-id:date:subject:cc:to\n :from:from:to:cc:subject:date:message-id:reply-to;\n bh=UHGu4JKP4YqC48fGuPT6SgGifyRfDyuFcFA/A5H5NIg=;\n b=bwsPC7GqjlVnlsZ9+4ZVLmf7JzIqZTmJXh+JYh+79Qb/EsZK7A3SoUq9YxqIXc2gho\n W/+d2C4QQz4/X/+7L9ugkB9/00KlZTGHOIDsBrvxG4sVpPqmngMYVaJnSJneKG/QjpMY\n QXk5RLQmBPS4oyXR38zzKMm2KvX0hX4Mv0/TM=", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=1e100.net; s=20230601; t=1703650909; x=1704255709;\n h=mime-version:references:in-reply-to:message-id:date:subject:cc:to\n :from:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to;\n bh=UHGu4JKP4YqC48fGuPT6SgGifyRfDyuFcFA/A5H5NIg=;\n b=CDNSGlBK21MTM1DIr4pSnPK0OL19TosOxBgIwyTwoZsRRl+BWTCcNM7UX8BpAmHOUD\n FAPFJkl5vn9PlsPq2LE7l/QS2nu2jQktv8U2aEq2tm4IbJ1s+ShHz65sjUJ6yFL7b8HE\n uSo0EBTJehZYORTn1S1Pbe0Ppi+GQKuL+RYpC3utBVlEUBbQ7CBN4Yrn5ktpJV/sa1q3\n AlEbCJYUkHBp1UNjoUU1ffm4Z8f8vkXZQEXYizHIRLVsCzdB+2G7jpsgbzB4YZ77jmXb\n mU8g4qzbYV8LuJAvkqFxP7Js+PYvs13UD5Ahkop01FzfUc4PdfQ3sq0207U8vbvqMtMi\n 0jTA==", "X-Gm-Message-State": "AOJu0YxP8/No4XHgNUKHH+FWxOOfg9F7Vvvs/K7GMPCZrNSDf816X4MG\n 4GKMM34WSqfRu+Ct5Xr7C5pajbhSE2waQ+8HEeMAUv8JW8EwNn6DUGuOs9/6gxjrPQQ+xqpEm7K\n st+jClpeUEuRnEOe6uOrZEhtcKCMI9vHN93/jwQ1fnKDtPug2vrr5DXya7K/VO6Id/nPZNfuG9/\n k=", "X-Google-Smtp-Source": "\n AGHT+IGTHrJMh3aW+nyEWgdK8Z76utsGAIoEP2WUdCOx5jQzmte844CYDyLOI2inizhU4gMHM7vg6g==", "X-Received": "by 2002:a05:622a:205:b0:425:a5fe:9b71 with SMTP id\n b5-20020a05622a020500b00425a5fe9b71mr12390128qtx.43.1703650908781;\n Tue, 26 Dec 2023 20:21:48 -0800 (PST)", "From": "Ajit Khaparde <ajit.khaparde@broadcom.com>", "To": "dev@dpdk.org", "Cc": "Damodharam Ammepalli <damodharam.ammepalli@broadcom.com>", "Subject": "[PATCH v3 17/18] net/bnxt: add AVX2 support for compressed CQE", "Date": "Tue, 26 Dec 2023 20:21:18 -0800", "Message-Id": "<20231227042119.72469-18-ajit.khaparde@broadcom.com>", "X-Mailer": "git-send-email 2.39.2 (Apple Git-143)", "In-Reply-To": "<20231227042119.72469-1-ajit.khaparde@broadcom.com>", "References": "<20231227042119.72469-1-ajit.khaparde@broadcom.com>", "MIME-Version": "1.0", "Content-Type": "multipart/signed; protocol=\"application/pkcs7-signature\";\n micalg=sha-256; boundary=\"000000000000447e9e060d762316\"", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "P7 device family supports 16 byte Rx completions.\nAdd AVX2 vector mode for compressed Rx CQE.\n\nSigned-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>\nReviewed-by: Damodharam Ammepalli <damodharam.ammepalli@broadcom.com>\n---\n drivers/net/bnxt/bnxt_ethdev.c | 5 +\n drivers/net/bnxt/bnxt_rxr.h | 2 +\n drivers/net/bnxt/bnxt_rxtx_vec_avx2.c | 309 ++++++++++++++++++++++++++\n 3 files changed, 316 insertions(+)", "diff": "diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c\nindex 031028eda1..bd8c7557dd 100644\n--- a/drivers/net/bnxt/bnxt_ethdev.c\n+++ b/drivers/net/bnxt/bnxt_ethdev.c\n@@ -1406,6 +1406,8 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev)\n \t\t\t \"Using AVX2 vector mode receive for port %d\\n\",\n \t\t\t eth_dev->data->port_id);\n \t\tbp->flags |= BNXT_FLAG_RX_VECTOR_PKT_MODE;\n+\t\tif (bnxt_compressed_rx_cqe_mode_enabled(bp))\n+\t\t\treturn bnxt_crx_pkts_vec_avx2;\n \t\treturn bnxt_recv_pkts_vec_avx2;\n \t}\n #endif\n@@ -3124,6 +3126,9 @@ static const struct {\n \t{bnxt_recv_pkts,\t\t\"Scalar\"},\n #if defined(RTE_ARCH_X86)\n \t{bnxt_recv_pkts_vec,\t\t\"Vector SSE\"},\n+#endif\n+#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)\n+\t{bnxt_crx_pkts_vec_avx2,\t\"Vector AVX2\"},\n \t{bnxt_recv_pkts_vec_avx2,\t\"Vector AVX2\"},\n #endif\n #if defined(RTE_ARCH_ARM64)\ndiff --git a/drivers/net/bnxt/bnxt_rxr.h b/drivers/net/bnxt/bnxt_rxr.h\nindex c51bb2d62c..a474a69ae3 100644\n--- a/drivers/net/bnxt/bnxt_rxr.h\n+++ b/drivers/net/bnxt/bnxt_rxr.h\n@@ -162,6 +162,8 @@ int bnxt_rxq_vec_setup(struct bnxt_rx_queue *rxq);\n #if defined(RTE_ARCH_X86)\n uint16_t bnxt_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,\n \t\t\t\t uint16_t nb_pkts);\n+uint16_t bnxt_crx_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,\n+\t\t\t\tuint16_t nb_pkts);\n #endif\n void bnxt_set_mark_in_mbuf(struct bnxt *bp,\n \t\t\t struct rx_pkt_cmpl_hi *rxcmp1,\ndiff --git a/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c b/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c\nindex ea8dbaffba..ce6b597611 100644\n--- a/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c\n+++ b/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c\n@@ -361,6 +361,294 @@ recv_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)\n \treturn nb_rx_pkts;\n }\n \n+static uint16_t\n+crx_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)\n+{\n+\tstruct bnxt_rx_queue *rxq = rx_queue;\n+\tconst __m256i mbuf_init =\n+\t\t_mm256_set_epi64x(0, 0, 0, rxq->mbuf_initializer);\n+\tstruct bnxt_cp_ring_info *cpr = rxq->cp_ring;\n+\tstruct bnxt_rx_ring_info *rxr = rxq->rx_ring;\n+\tuint16_t cp_ring_size = cpr->cp_ring_struct->ring_size;\n+\tuint16_t rx_ring_size = rxr->rx_ring_struct->ring_size;\n+\tstruct cmpl_base *cp_desc_ring = cpr->cp_desc_ring;\n+\tuint64_t valid, desc_valid_mask = ~0ULL;\n+\tconst __m256i info3_v_mask = _mm256_set1_epi32(CMPL_BASE_V);\n+\tuint32_t raw_cons = cpr->cp_raw_cons;\n+\tuint32_t cons, mbcons;\n+\tint nb_rx_pkts = 0;\n+\tint i;\n+\tconst __m256i valid_target =\n+\t\t_mm256_set1_epi32(!!(raw_cons & cp_ring_size));\n+\tconst __m256i shuf_msk =\n+\t\t_mm256_set_epi8(15, 14, 13, 12, /* rss */\n+\t\t\t\t7, 6, /* vlan_tci */\n+\t\t\t\t3, 2, /* data_len */\n+\t\t\t\t0xFF, 0xFF, 3, 2, /* pkt_len */\n+\t\t\t\t0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type (zeroes) */\n+\t\t\t\t15, 14, 13, 12, /* rss */\n+\t\t\t\t7, 6, /* vlan_tci */\n+\t\t\t\t3, 2, /* data_len */\n+\t\t\t\t0xFF, 0xFF, 3, 2, /* pkt_len */\n+\t\t\t\t0xFF, 0xFF, 0xFF, 0xFF); /* pkt_type (zeroes) */\n+\tconst __m256i flags_type_mask =\n+\t\t_mm256_set1_epi32(RX_PKT_CMPL_FLAGS_ITYPE_MASK);\n+\tconst __m256i flags2_mask1 =\n+\t\t_mm256_set1_epi32(CMPL_FLAGS2_VLAN_TUN_MSK);\n+\tconst __m256i flags2_mask2 =\n+\t\t_mm256_set1_epi32(RX_PKT_CMPL_FLAGS2_IP_TYPE);\n+\tconst __m256i rss_mask =\n+\t\t_mm256_set1_epi32(RX_PKT_CMPL_FLAGS_RSS_VALID);\n+\t__m256i t0, t1, flags_type, flags2, index, errors;\n+\t__m256i ptype_idx, ptypes, is_tunnel;\n+\t__m256i mbuf01, mbuf23, mbuf45, mbuf67;\n+\t__m256i rearm0, rearm1, rearm2, rearm3, rearm4, rearm5, rearm6, rearm7;\n+\t__m256i ol_flags, ol_flags_hi;\n+\t__m256i rss_flags;\n+\n+\t/* Validate ptype table indexing at build time. */\n+\tbnxt_check_ptype_constants();\n+\n+\t/* If Rx Q was stopped return */\n+\tif (unlikely(!rxq->rx_started))\n+\t\treturn 0;\n+\n+\tif (rxq->rxrearm_nb >= rxq->rx_free_thresh)\n+\t\tbnxt_rxq_rearm(rxq, rxr);\n+\n+\tnb_pkts = RTE_ALIGN_FLOOR(nb_pkts, BNXT_RX_DESCS_PER_LOOP_VEC256);\n+\n+\tcons = raw_cons & (cp_ring_size - 1);\n+\tmbcons = raw_cons & (rx_ring_size - 1);\n+\n+\t/* Return immediately if there is not at least one completed packet. */\n+\tif (!bnxt_cpr_cmp_valid(&cp_desc_ring[cons], raw_cons, cp_ring_size))\n+\t\treturn 0;\n+\n+\t/* Ensure that we do not go past the ends of the rings. */\n+\tnb_pkts = RTE_MIN(nb_pkts, RTE_MIN(rx_ring_size - mbcons,\n+\t\t\t\t\t cp_ring_size - cons));\n+\t/*\n+\t * If we are at the end of the ring, ensure that descriptors after the\n+\t * last valid entry are not treated as valid. Otherwise, force the\n+\t * maximum number of packets to receive to be a multiple of the per-\n+\t * loop count.\n+\t */\n+\tif (nb_pkts < BNXT_RX_DESCS_PER_LOOP_VEC256) {\n+\t\tdesc_valid_mask >>=\n+\t\t\tCHAR_BIT * (BNXT_RX_DESCS_PER_LOOP_VEC256 - nb_pkts);\n+\t} else {\n+\t\tnb_pkts =\n+\t\t\tRTE_ALIGN_FLOOR(nb_pkts, BNXT_RX_DESCS_PER_LOOP_VEC256);\n+\t}\n+\n+\t/* Handle RX burst request */\n+\tfor (i = 0; i < nb_pkts; i += BNXT_RX_DESCS_PER_LOOP_VEC256,\n+\t\t\t\t cons += BNXT_RX_DESCS_PER_LOOP_VEC256,\n+\t\t\t\t mbcons += BNXT_RX_DESCS_PER_LOOP_VEC256) {\n+\t\t__m256i rxcmp0_1, rxcmp2_3, rxcmp4_5, rxcmp6_7, info3_v;\n+\t\t__m256i errors_v2;\n+\t\tuint32_t num_valid;\n+\n+\t\t/* Copy eight mbuf pointers to output array. */\n+\t\tt0 = _mm256_loadu_si256((void *)&rxr->rx_buf_ring[mbcons]);\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i], t0);\n+#ifdef RTE_ARCH_X86_64\n+\t\tt0 = _mm256_loadu_si256((void *)&rxr->rx_buf_ring[mbcons + 4]);\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 4], t0);\n+#endif\n+\n+\t\t/*\n+\t\t * Load eight receive completion descriptors into 256-bit\n+\t\t * registers. Loads are issued in reverse order in order to\n+\t\t * ensure consistent state.\n+\t\t */\n+\t\trxcmp6_7 = _mm256_loadu_si256((void *)&cp_desc_ring[cons + 6]);\n+\t\trte_compiler_barrier();\n+\t\trxcmp4_5 = _mm256_loadu_si256((void *)&cp_desc_ring[cons + 4]);\n+\t\trte_compiler_barrier();\n+\t\trxcmp2_3 = _mm256_loadu_si256((void *)&cp_desc_ring[cons + 2]);\n+\t\trte_compiler_barrier();\n+\t\trxcmp0_1 = _mm256_loadu_si256((void *)&cp_desc_ring[cons + 0]);\n+\n+\t\t/* Compute packet type table indices for eight packets. */\n+\t\tt0 = _mm256_unpacklo_epi32(rxcmp0_1, rxcmp2_3);\n+\t\tt1 = _mm256_unpacklo_epi32(rxcmp4_5, rxcmp6_7);\n+\t\tflags_type = _mm256_unpacklo_epi64(t0, t1);\n+\t\tptype_idx = _mm256_and_si256(flags_type, flags_type_mask);\n+\t\tptype_idx = _mm256_srli_epi32(ptype_idx,\n+\t\t\t\t\t RX_PKT_CMPL_FLAGS_ITYPE_SFT -\n+\t\t\t\t\t BNXT_PTYPE_TBL_TYPE_SFT);\n+\n+\t\tt0 = _mm256_unpacklo_epi32(rxcmp0_1, rxcmp2_3);\n+\t\tt1 = _mm256_unpacklo_epi32(rxcmp4_5, rxcmp6_7);\n+\t\tflags2 = _mm256_unpackhi_epi64(t0, t1);\n+\n+\t\tt0 = _mm256_srli_epi32(_mm256_and_si256(flags2, flags2_mask1),\n+\t\t\t\t RX_PKT_CMPL_FLAGS2_META_FORMAT_SFT -\n+\t\t\t\t BNXT_PTYPE_TBL_VLAN_SFT);\n+\t\tptype_idx = _mm256_or_si256(ptype_idx, t0);\n+\n+\t\tt0 = _mm256_srli_epi32(_mm256_and_si256(flags2, flags2_mask2),\n+\t\t\t\t RX_PKT_CMPL_FLAGS2_IP_TYPE_SFT -\n+\t\t\t\t BNXT_PTYPE_TBL_IP_VER_SFT);\n+\t\tptype_idx = _mm256_or_si256(ptype_idx, t0);\n+\n+\t\t/*\n+\t\t * Load ptypes for eight packets using gather. Gather operations\n+\t\t * have extremely high latency (~19 cycles), execution and use\n+\t\t * of result should be separated as much as possible.\n+\t\t */\n+\t\tptypes = _mm256_i32gather_epi32((int *)bnxt_ptype_table,\n+\t\t\t\t\t\tptype_idx, sizeof(uint32_t));\n+\t\t/*\n+\t\t * Compute ol_flags and checksum error table indices for eight\n+\t\t * packets.\n+\t\t */\n+\t\tis_tunnel = _mm256_and_si256(flags2, _mm256_set1_epi32(4));\n+\t\tis_tunnel = _mm256_slli_epi32(is_tunnel, 3);\n+\t\tflags2 = _mm256_and_si256(flags2, _mm256_set1_epi32(0x1F));\n+\n+\t\t/* Extract errors_v2 fields for eight packets. */\n+\t\tt0 = _mm256_unpackhi_epi32(rxcmp0_1, rxcmp2_3);\n+\t\tt1 = _mm256_unpackhi_epi32(rxcmp4_5, rxcmp6_7);\n+\t\terrors_v2 = _mm256_unpacklo_epi64(t0, t1);\n+\n+\t\terrors = _mm256_srli_epi32(errors_v2, 4);\n+\t\terrors = _mm256_and_si256(errors, _mm256_set1_epi32(0xF));\n+\t\terrors = _mm256_and_si256(errors, flags2);\n+\n+\t\tindex = _mm256_andnot_si256(errors, flags2);\n+\t\terrors = _mm256_or_si256(errors,\n+\t\t\t\t\t _mm256_srli_epi32(is_tunnel, 1));\n+\t\tindex = _mm256_or_si256(index, is_tunnel);\n+\n+\t\t/*\n+\t\t * Load ol_flags for eight packets using gather. Gather\n+\t\t * operations have extremely high latency (~19 cycles),\n+\t\t * execution and use of result should be separated as much\n+\t\t * as possible.\n+\t\t */\n+\t\tol_flags = _mm256_i32gather_epi32((int *)rxr->ol_flags_table,\n+\t\t\t\t\t\t index, sizeof(uint32_t));\n+\t\terrors = _mm256_i32gather_epi32((int *)rxr->ol_flags_err_table,\n+\t\t\t\t\t\terrors, sizeof(uint32_t));\n+\n+\t\t/*\n+\t\t * Pack the 128-bit array of valid descriptor flags into 64\n+\t\t * bits and count the number of set bits in order to determine\n+\t\t * the number of valid descriptors.\n+\t\t */\n+\t\tconst __m256i perm_msk =\n+\t\t\t\t_mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);\n+\t\tinfo3_v = _mm256_permutevar8x32_epi32(errors_v2, perm_msk);\n+\t\tinfo3_v = _mm256_and_si256(errors_v2, info3_v_mask);\n+\t\tinfo3_v = _mm256_xor_si256(info3_v, valid_target);\n+\n+\t\tinfo3_v = _mm256_packs_epi32(info3_v, _mm256_setzero_si256());\n+\t\tvalid = _mm_cvtsi128_si64(_mm256_extracti128_si256(info3_v, 1));\n+\t\tvalid = (valid << CHAR_BIT) |\n+\t\t\t_mm_cvtsi128_si64(_mm256_castsi256_si128(info3_v));\n+\t\tnum_valid = rte_popcount64(valid & desc_valid_mask);\n+\n+\t\tif (num_valid == 0)\n+\t\t\tbreak;\n+\n+\t\t/* Update mbuf rearm_data for eight packets. */\n+\t\tmbuf01 = _mm256_shuffle_epi8(rxcmp0_1, shuf_msk);\n+\t\tmbuf23 = _mm256_shuffle_epi8(rxcmp2_3, shuf_msk);\n+\t\tmbuf45 = _mm256_shuffle_epi8(rxcmp4_5, shuf_msk);\n+\t\tmbuf67 = _mm256_shuffle_epi8(rxcmp6_7, shuf_msk);\n+\n+\t\t/* Blend in ptype field for two mbufs at a time. */\n+\t\tmbuf01 = _mm256_blend_epi32(mbuf01, ptypes, 0x11);\n+\t\tmbuf23 = _mm256_blend_epi32(mbuf23,\n+\t\t\t\t\t_mm256_srli_si256(ptypes, 4), 0x11);\n+\t\tmbuf45 = _mm256_blend_epi32(mbuf45,\n+\t\t\t\t\t_mm256_srli_si256(ptypes, 8), 0x11);\n+\t\tmbuf67 = _mm256_blend_epi32(mbuf67,\n+\t\t\t\t\t_mm256_srli_si256(ptypes, 12), 0x11);\n+\n+\t\t/* Unpack rearm data, set fixed fields for first four mbufs. */\n+\t\trearm0 = _mm256_permute2f128_si256(mbuf_init, mbuf01, 0x20);\n+\t\trearm1 = _mm256_blend_epi32(mbuf_init, mbuf01, 0xF0);\n+\t\trearm2 = _mm256_permute2f128_si256(mbuf_init, mbuf23, 0x20);\n+\t\trearm3 = _mm256_blend_epi32(mbuf_init, mbuf23, 0xF0);\n+\n+\t\t/* Compute final ol_flags values for eight packets. */\n+\t\trss_flags = _mm256_and_si256(flags_type, rss_mask);\n+\t\trss_flags = _mm256_srli_epi32(rss_flags, 9);\n+\t\tol_flags = _mm256_or_si256(ol_flags, errors);\n+\t\tol_flags = _mm256_or_si256(ol_flags, rss_flags);\n+\t\tol_flags_hi = _mm256_permute2f128_si256(ol_flags,\n+\t\t\t\t\t\t\tol_flags, 0x11);\n+\n+\t\t/* Set ol_flags fields for first four packets. */\n+\t\trearm0 = _mm256_blend_epi32(rearm0,\n+\t\t\t\t\t _mm256_slli_si256(ol_flags, 8),\n+\t\t\t\t\t 0x04);\n+\t\trearm1 = _mm256_blend_epi32(rearm1,\n+\t\t\t\t\t _mm256_slli_si256(ol_flags_hi, 8),\n+\t\t\t\t\t 0x04);\n+\t\trearm2 = _mm256_blend_epi32(rearm2,\n+\t\t\t\t\t _mm256_slli_si256(ol_flags, 4),\n+\t\t\t\t\t 0x04);\n+\t\trearm3 = _mm256_blend_epi32(rearm3,\n+\t\t\t\t\t _mm256_slli_si256(ol_flags_hi, 4),\n+\t\t\t\t\t 0x04);\n+\n+\t\t/* Store all mbuf fields for first four packets. */\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 0]->rearm_data,\n+\t\t\t\t rearm0);\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 1]->rearm_data,\n+\t\t\t\t rearm1);\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 2]->rearm_data,\n+\t\t\t\t rearm2);\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 3]->rearm_data,\n+\t\t\t\t rearm3);\n+\n+\t\t/* Unpack rearm data, set fixed fields for final four mbufs. */\n+\t\trearm4 = _mm256_permute2f128_si256(mbuf_init, mbuf45, 0x20);\n+\t\trearm5 = _mm256_blend_epi32(mbuf_init, mbuf45, 0xF0);\n+\t\trearm6 = _mm256_permute2f128_si256(mbuf_init, mbuf67, 0x20);\n+\t\trearm7 = _mm256_blend_epi32(mbuf_init, mbuf67, 0xF0);\n+\n+\t\t/* Set ol_flags fields for final four packets. */\n+\t\trearm4 = _mm256_blend_epi32(rearm4, ol_flags, 0x04);\n+\t\trearm5 = _mm256_blend_epi32(rearm5, ol_flags_hi, 0x04);\n+\t\trearm6 = _mm256_blend_epi32(rearm6,\n+\t\t\t\t\t _mm256_srli_si256(ol_flags, 4),\n+\t\t\t\t\t 0x04);\n+\t\trearm7 = _mm256_blend_epi32(rearm7,\n+\t\t\t\t\t _mm256_srli_si256(ol_flags_hi, 4),\n+\t\t\t\t\t 0x04);\n+\n+\t\t/* Store all mbuf fields for final four packets. */\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 4]->rearm_data,\n+\t\t\t\t rearm4);\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 5]->rearm_data,\n+\t\t\t\t rearm5);\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 6]->rearm_data,\n+\t\t\t\t rearm6);\n+\t\t_mm256_storeu_si256((void *)&rx_pkts[i + 7]->rearm_data,\n+\t\t\t\t rearm7);\n+\n+\t\tnb_rx_pkts += num_valid;\n+\t\tif (num_valid < BNXT_RX_DESCS_PER_LOOP_VEC256)\n+\t\t\tbreak;\n+\t}\n+\n+\tif (nb_rx_pkts) {\n+\t\trxr->rx_raw_prod = RING_ADV(rxr->rx_raw_prod, nb_rx_pkts);\n+\n+\t\trxq->rxrearm_nb += nb_rx_pkts;\n+\t\tcpr->cp_raw_cons += nb_rx_pkts;\n+\t\tbnxt_db_cq(cpr);\n+\t}\n+\n+\treturn nb_rx_pkts;\n+}\n+\n uint16_t\n bnxt_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,\n \t\t\t uint16_t nb_pkts)\n@@ -382,6 +670,27 @@ bnxt_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,\n \treturn cnt + recv_burst_vec_avx2(rx_queue, rx_pkts + cnt, nb_pkts);\n }\n \n+uint16_t\n+bnxt_crx_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,\n+\t\t uint16_t nb_pkts)\n+{\n+\tuint16_t cnt = 0;\n+\n+\twhile (nb_pkts > RTE_BNXT_MAX_RX_BURST) {\n+\t\tuint16_t burst;\n+\n+\t\tburst = crx_burst_vec_avx2(rx_queue, rx_pkts + cnt,\n+\t\t\t\t\t RTE_BNXT_MAX_RX_BURST);\n+\n+\t\tcnt += burst;\n+\t\tnb_pkts -= burst;\n+\n+\t\tif (burst < RTE_BNXT_MAX_RX_BURST)\n+\t\t\treturn cnt;\n+\t}\n+\treturn cnt + crx_burst_vec_avx2(rx_queue, rx_pkts + cnt, nb_pkts);\n+}\n+\n static void\n bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq)\n {\n", "prefixes": [ "v3", "17/18" ] }