Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/66681/?format=api
{ "id": 66681, "url": "http://patches.dpdk.org/api/patches/66681/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20200316074603.10998-7-leyi.rong@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20200316074603.10998-7-leyi.rong@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20200316074603.10998-7-leyi.rong@intel.com", "date": "2020-03-16T07:45:57", "name": "[06/12] net/iavf: flexible Rx descriptor support in SSE path", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "6b86c3d4036a63cc10162aaa04e230a8f5d398e5", "submitter": { "id": 1204, "url": "http://patches.dpdk.org/api/people/1204/?format=api", "name": "Leyi Rong", "email": "leyi.rong@intel.com" }, "delegate": { "id": 31221, "url": "http://patches.dpdk.org/api/users/31221/?format=api", "username": "yexl", "first_name": "xiaolong", "last_name": "ye", "email": "xiaolong.ye@intel.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20200316074603.10998-7-leyi.rong@intel.com/mbox/", "series": [ { "id": 8918, "url": "http://patches.dpdk.org/api/series/8918/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=8918", "date": "2020-03-16T07:45:51", "name": "framework for advanced iAVF PMD", "version": 1, "mbox": "http://patches.dpdk.org/series/8918/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/66681/comments/", "check": "fail", "checks": "http://patches.dpdk.org/api/patches/66681/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 3749DA0559;\n\tMon, 16 Mar 2020 08:58:37 +0100 (CET)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id DAB571C0C3;\n\tMon, 16 Mar 2020 08:57:36 +0100 (CET)", "from mga18.intel.com (mga18.intel.com [134.134.136.126])\n by dpdk.org (Postfix) with ESMTP id 075481C0AC\n for <dev@dpdk.org>; Mon, 16 Mar 2020 08:57:32 +0100 (CET)", "from orsmga004.jf.intel.com ([10.7.209.38])\n by orsmga106.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 16 Mar 2020 00:57:32 -0700", "from dpdk-lrong-srv-04.sh.intel.com ([10.67.119.221])\n by orsmga004.jf.intel.com with ESMTP; 16 Mar 2020 00:57:31 -0700" ], "IronPort-SDR": [ "\n xUaPdymivkrbQbSmGhCXFbRBTCKvqxDTBcPLt+K7IP8K7SxPWSqy+LDTHaneFJMo/QdGxWThp1\n zr7gFttUoHow==", "\n xJCUJtXWU6zFVZL7y2Zgcv9SLX2Ue1415yoVPE6/dAPG+mpfEsS/JEYJ5uiS7qbZCeDRw3qMSm\n RKYhtHbtr+yg==" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.70,559,1574150400\"; d=\"scan'208\";a=\"390622521\"", "From": "Leyi Rong <leyi.rong@intel.com>", "To": "qi.z.zhang@intel.com,\n\txiaolong.ye@intel.com", "Cc": "dev@dpdk.org,\n\tLeyi Rong <leyi.rong@intel.com>", "Date": "Mon, 16 Mar 2020 15:45:57 +0800", "Message-Id": "<20200316074603.10998-7-leyi.rong@intel.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20200316074603.10998-1-leyi.rong@intel.com>", "References": "<20200316074603.10998-1-leyi.rong@intel.com>", "Subject": "[dpdk-dev] [PATCH 06/12] net/iavf: flexible Rx descriptor support\n\tin SSE path", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Support flexible Rx descriptor format in SSE\npath of iAVF PMD.\n\nSigned-off-by: Leyi Rong <leyi.rong@intel.com>\n---\n drivers/net/iavf/iavf_rxtx.c | 4 +-\n drivers/net/iavf/iavf_rxtx.h | 5 +\n drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++\n 3 files changed, 421 insertions(+), 2 deletions(-)", "diff": "diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c\nindex 57fffece9..5d484d5e9 100644\n--- a/drivers/net/iavf/iavf_rxtx.c\n+++ b/drivers/net/iavf/iavf_rxtx.c\n@@ -2119,7 +2119,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)\n \t\t\t\tVIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)\n \t\t\t\tdev->rx_pkt_burst = use_avx2 ?\n \t\t\t\t\tiavf_recv_scattered_pkts_vec_avx2_flex_rxd :\n-\t\t\t\t\tiavf_recv_scattered_pkts_vec;\n+\t\t\t\t\tiavf_recv_scattered_pkts_vec_flex_rxd;\n \t\t\telse\n \t\t\t\tdev->rx_pkt_burst = use_avx2 ?\n \t\t\t\t\tiavf_recv_scattered_pkts_vec_avx2 :\n@@ -2132,7 +2132,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)\n \t\t\t\tVIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)\n \t\t\t\tdev->rx_pkt_burst = use_avx2 ?\n \t\t\t\t\tiavf_recv_pkts_vec_avx2_flex_rxd :\n-\t\t\t\t\tiavf_recv_pkts_vec;\n+\t\t\t\t\tiavf_recv_pkts_vec_flex_rxd;\n \t\t\telse\n \t\t\t\tdev->rx_pkt_burst = use_avx2 ?\n \t\t\t\t\tiavf_recv_pkts_vec_avx2 :\ndiff --git a/drivers/net/iavf/iavf_rxtx.h b/drivers/net/iavf/iavf_rxtx.h\nindex de13fd516..c85207dae 100644\n--- a/drivers/net/iavf/iavf_rxtx.h\n+++ b/drivers/net/iavf/iavf_rxtx.h\n@@ -205,9 +205,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);\n \n uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,\n \t\t\t uint16_t nb_pkts);\n+uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,\n+\t\t\t\t uint16_t nb_pkts);\n uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,\n \t\t\t\t struct rte_mbuf **rx_pkts,\n \t\t\t\t uint16_t nb_pkts);\n+uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,\n+\t\t\t\t\t struct rte_mbuf **rx_pkts,\n+\t\t\t\t\t uint16_t nb_pkts);\n uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,\n \t\t\t\t uint16_t nb_pkts);\n uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,\ndiff --git a/drivers/net/iavf/iavf_rxtx_vec_sse.c b/drivers/net/iavf/iavf_rxtx_vec_sse.c\nindex 0365c49e1..9c1f2a445 100644\n--- a/drivers/net/iavf/iavf_rxtx_vec_sse.c\n+++ b/drivers/net/iavf/iavf_rxtx_vec_sse.c\n@@ -189,6 +189,109 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],\n \t_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);\n }\n \n+static inline void\n+flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],\n+\t\t struct rte_mbuf **rx_pkts)\n+{\n+\tconst __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer);\n+\t__m128i rearm0, rearm1, rearm2, rearm3;\n+\n+\t__m128i tmp_desc, flags, rss_vlan;\n+\n+\t/* mask everything except checksum, RSS and VLAN flags.\n+\t * bit6:4 for checksum.\n+\t * bit12 for RSS indication.\n+\t * bit13 for VLAN indication.\n+\t */\n+\tconst __m128i desc_mask = _mm_set_epi32(0x3070, 0x3070,\n+\t\t\t\t\t\t0x3070, 0x3070);\n+\n+\tconst __m128i cksum_mask = _mm_set_epi32(PKT_RX_IP_CKSUM_MASK |\n+\t\t\t\t\t\t PKT_RX_L4_CKSUM_MASK |\n+\t\t\t\t\t\t PKT_RX_EIP_CKSUM_BAD,\n+\t\t\t\t\t\t PKT_RX_IP_CKSUM_MASK |\n+\t\t\t\t\t\t PKT_RX_L4_CKSUM_MASK |\n+\t\t\t\t\t\t PKT_RX_EIP_CKSUM_BAD,\n+\t\t\t\t\t\t PKT_RX_IP_CKSUM_MASK |\n+\t\t\t\t\t\t PKT_RX_L4_CKSUM_MASK |\n+\t\t\t\t\t\t PKT_RX_EIP_CKSUM_BAD,\n+\t\t\t\t\t\t PKT_RX_IP_CKSUM_MASK |\n+\t\t\t\t\t\t PKT_RX_L4_CKSUM_MASK |\n+\t\t\t\t\t\t PKT_RX_EIP_CKSUM_BAD);\n+\n+\t/* map the checksum, rss and vlan fields to the checksum, rss\n+\t * and vlan flag\n+\t */\n+\tconst __m128i cksum_flags = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,\n+\t\t\t/* shift right 1 bit to make sure it not exceed 255 */\n+\t\t\t(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD |\n+\t\t\t PKT_RX_IP_CKSUM_BAD) >> 1,\n+\t\t\t(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD |\n+\t\t\t PKT_RX_IP_CKSUM_GOOD) >> 1,\n+\t\t\t(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD |\n+\t\t\t PKT_RX_IP_CKSUM_BAD) >> 1,\n+\t\t\t(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD |\n+\t\t\t PKT_RX_IP_CKSUM_GOOD) >> 1,\n+\t\t\t(PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1,\n+\t\t\t(PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_GOOD) >> 1,\n+\t\t\t(PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD) >> 1,\n+\t\t\t(PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_GOOD) >> 1);\n+\n+\tconst __m128i rss_vlan_flags = _mm_set_epi8(0, 0, 0, 0,\n+\t\t\t0, 0, 0, 0,\n+\t\t\t0, 0, 0, 0,\n+\t\t\tPKT_RX_RSS_HASH | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,\n+\t\t\tPKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,\n+\t\t\tPKT_RX_RSS_HASH, 0);\n+\n+\t/* merge 4 descriptors */\n+\tflags = _mm_unpackhi_epi32(descs[0], descs[1]);\n+\ttmp_desc = _mm_unpackhi_epi32(descs[2], descs[3]);\n+\ttmp_desc = _mm_unpacklo_epi64(flags, tmp_desc);\n+\ttmp_desc = _mm_and_si128(flags, desc_mask);\n+\n+\t/* checksum flags */\n+\ttmp_desc = _mm_srli_epi32(tmp_desc, 4);\n+\tflags = _mm_shuffle_epi8(cksum_flags, tmp_desc);\n+\t/* then we shift left 1 bit */\n+\tflags = _mm_slli_epi32(flags, 1);\n+\t/* we need to mask out the reduntant bits introduced by RSS or\n+\t * VLAN fields.\n+\t */\n+\tflags = _mm_and_si128(flags, cksum_mask);\n+\n+\t/* RSS, VLAN flag */\n+\ttmp_desc = _mm_srli_epi32(tmp_desc, 8);\n+\trss_vlan = _mm_shuffle_epi8(rss_vlan_flags, tmp_desc);\n+\n+\t/* merge the flags */\n+\tflags = _mm_or_si128(flags, rss_vlan);\n+\n+\t/**\n+\t * At this point, we have the 4 sets of flags in the low 16-bits\n+\t * of each 32-bit value in flags.\n+\t * We want to extract these, and merge them with the mbuf init data\n+\t * so we can do a single 16-byte write to the mbuf to set the flags\n+\t * and all the other initialization fields. Extracting the\n+\t * appropriate flags means that we have to do a shift and blend for\n+\t * each mbuf before we do the write.\n+\t */\n+\trearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(flags, 8), 0x10);\n+\trearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(flags, 4), 0x10);\n+\trearm2 = _mm_blend_epi16(mbuf_init, flags, 0x10);\n+\trearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(flags, 4), 0x10);\n+\n+\t/* write the rearm data and the olflags in one write */\n+\tRTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=\n+\t\t\t offsetof(struct rte_mbuf, rearm_data) + 8);\n+\tRTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=\n+\t\t\t RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));\n+\t_mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0);\n+\t_mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1);\n+\t_mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2);\n+\t_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);\n+}\n+\n #define PKTLEN_SHIFT 10\n \n static inline void\n@@ -207,6 +310,26 @@ desc_to_ptype_v(__m128i descs[4], struct rte_mbuf **rx_pkts,\n \trx_pkts[3]->packet_type = type_table[_mm_extract_epi8(ptype1, 8)];\n }\n \n+static inline void\n+flex_desc_to_ptype_v(__m128i descs[4], struct rte_mbuf **rx_pkts,\n+\t\t const uint32_t *type_table)\n+{\n+\tconst __m128i ptype_mask = _mm_set_epi16(0, IAVF_RX_FLEX_DESC_PTYPE_M,\n+\t\t\t\t\t\t 0, IAVF_RX_FLEX_DESC_PTYPE_M,\n+\t\t\t\t\t\t 0, IAVF_RX_FLEX_DESC_PTYPE_M,\n+\t\t\t\t\t\t 0, IAVF_RX_FLEX_DESC_PTYPE_M);\n+\t__m128i ptype_01 = _mm_unpacklo_epi32(descs[0], descs[1]);\n+\t__m128i ptype_23 = _mm_unpacklo_epi32(descs[2], descs[3]);\n+\t__m128i ptype_all = _mm_unpacklo_epi64(ptype_01, ptype_23);\n+\n+\tptype_all = _mm_and_si128(ptype_all, ptype_mask);\n+\n+\trx_pkts[0]->packet_type = type_table[_mm_extract_epi16(ptype_all, 1)];\n+\trx_pkts[1]->packet_type = type_table[_mm_extract_epi16(ptype_all, 3)];\n+\trx_pkts[2]->packet_type = type_table[_mm_extract_epi16(ptype_all, 5)];\n+\trx_pkts[3]->packet_type = type_table[_mm_extract_epi16(ptype_all, 7)];\n+}\n+\n /* Notice:\n * - nb_pkts < IAVF_VPMD_DESCS_PER_LOOP, just return no packet\n * - nb_pkts > IAVF_VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST\n@@ -455,6 +578,243 @@ _recv_raw_pkts_vec(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts,\n \treturn nb_pkts_recd;\n }\n \n+/* Notice:\n+ * - nb_pkts < IAVF_VPMD_DESCS_PER_LOOP, just return no packet\n+ * - nb_pkts > IAVF_VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST\n+ * numbers of DD bits\n+ */\n+static inline uint16_t\n+_recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,\n+\t\t\t struct rte_mbuf **rx_pkts,\n+\t\t\t uint16_t nb_pkts, uint8_t *split_packet)\n+{\n+\tvolatile union iavf_rx_flex_desc *rxdp;\n+\tstruct rte_mbuf **sw_ring;\n+\tuint16_t nb_pkts_recd;\n+\tint pos;\n+\tuint64_t var;\n+\tconst uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;\n+\t__m128i crc_adjust = _mm_set_epi16\n+\t\t\t\t(0, 0, 0, /* ignore non-length fields */\n+\t\t\t\t -rxq->crc_len, /* sub crc on data_len */\n+\t\t\t\t 0, /* ignore high-16bits of pkt_len */\n+\t\t\t\t -rxq->crc_len, /* sub crc on pkt_len */\n+\t\t\t\t 0, 0 /* ignore pkt_type field */\n+\t\t\t\t);\n+\tconst __m128i zero = _mm_setzero_si128();\n+\t/* mask to shuffle from desc. to mbuf */\n+\tconst __m128i shuf_msk = _mm_set_epi8\n+\t\t\t(15, 14, 13, 12, /* octet 12~15, 32 bits rss */\n+\t\t\t 11, 10, /* octet 10~11, 16 bits vlan_macip */\n+\t\t\t 5, 4, /* octet 4~5, 16 bits data_len */\n+\t\t\t 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */\n+\t\t\t 5, 4, /* octet 4~5, low 16 bits pkt_len */\n+\t\t\t 0xFF, 0xFF, /* pkt_type set as unknown */\n+\t\t\t 0xFF, 0xFF /* pkt_type set as unknown */\n+\t\t\t);\n+\tconst __m128i eop_shuf_mask = _mm_set_epi8(0xFF, 0xFF,\n+\t\t\t\t\t\t 0xFF, 0xFF,\n+\t\t\t\t\t\t 0xFF, 0xFF,\n+\t\t\t\t\t\t 0xFF, 0xFF,\n+\t\t\t\t\t\t 0xFF, 0xFF,\n+\t\t\t\t\t\t 0xFF, 0xFF,\n+\t\t\t\t\t\t 0x04, 0x0C,\n+\t\t\t\t\t\t 0x00, 0x08);\n+\n+\t/**\n+\t * compile-time check the above crc_adjust layout is correct.\n+\t * NOTE: the first field (lowest address) is given last in set_epi16\n+\t * call above.\n+\t */\n+\tRTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=\n+\t\t\t offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);\n+\tRTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=\n+\t\t\t offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);\n+\n+\t/* 4 packets DD mask */\n+\tconst __m128i dd_check = _mm_set_epi64x(0x0000000100000001LL,\n+\t\t\t\t\t\t0x0000000100000001LL);\n+\t/* 4 packets EOP mask */\n+\tconst __m128i eop_check = _mm_set_epi64x(0x0000000200000002LL,\n+\t\t\t\t\t\t 0x0000000200000002LL);\n+\n+\t/* nb_pkts shall be less equal than IAVF_VPMD_RX_MAX_BURST */\n+\tnb_pkts = RTE_MIN(nb_pkts, IAVF_VPMD_RX_MAX_BURST);\n+\n+\t/* nb_pkts has to be floor-aligned to IAVF_VPMD_DESCS_PER_LOOP */\n+\tnb_pkts = RTE_ALIGN_FLOOR(nb_pkts, IAVF_VPMD_DESCS_PER_LOOP);\n+\n+\t/* Just the act of getting into the function from the application is\n+\t * going to cost about 7 cycles\n+\t */\n+\trxdp = (union iavf_rx_flex_desc *)rxq->rx_ring + rxq->rx_tail;\n+\n+\trte_prefetch0(rxdp);\n+\n+\t/* See if we need to rearm the RX queue - gives the prefetch a bit\n+\t * of time to act\n+\t */\n+\tif (rxq->rxrearm_nb > rxq->rx_free_thresh)\n+\t\tiavf_rxq_rearm(rxq);\n+\n+\t/* Before we start moving massive data around, check to see if\n+\t * there is actually a packet available\n+\t */\n+\tif (!(rxdp->wb.status_error0 &\n+\t rte_cpu_to_le_32(1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S)))\n+\t\treturn 0;\n+\n+\t/**\n+\t * Compile-time verify the shuffle mask\n+\t * NOTE: some field positions already verified above, but duplicated\n+\t * here for completeness in case of future modifications.\n+\t */\n+\tRTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=\n+\t\t\t offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);\n+\tRTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=\n+\t\t\t offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);\n+\tRTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=\n+\t\t\t offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);\n+\tRTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=\n+\t\t\t offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);\n+\n+\t/* Cache is empty -> need to scan the buffer rings, but first move\n+\t * the next 'n' mbufs into the cache\n+\t */\n+\tsw_ring = &rxq->sw_ring[rxq->rx_tail];\n+\n+\t/* A. load 4 packet in one loop\n+\t * [A*. mask out 4 unused dirty field in desc]\n+\t * B. copy 4 mbuf point from swring to rx_pkts\n+\t * C. calc the number of DD bits among the 4 packets\n+\t * [C*. extract the end-of-packet bit, if requested]\n+\t * D. fill info. from desc to mbuf\n+\t */\n+\n+\tfor (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;\n+\t pos += IAVF_VPMD_DESCS_PER_LOOP,\n+\t rxdp += IAVF_VPMD_DESCS_PER_LOOP) {\n+\t\t__m128i descs[IAVF_VPMD_DESCS_PER_LOOP];\n+\t\t__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;\n+\t\t__m128i staterr, sterr_tmp1, sterr_tmp2;\n+\t\t/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */\n+\t\t__m128i mbp1;\n+#if defined(RTE_ARCH_X86_64)\n+\t\t__m128i mbp2;\n+#endif\n+\n+\t\t/* B.1 load 2 (64 bit) or 4 (32 bit) mbuf points */\n+\t\tmbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]);\n+\t\t/* Read desc statuses backwards to avoid race condition */\n+\t\t/* A.1 load 4 pkts desc */\n+\t\tdescs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));\n+\t\trte_compiler_barrier();\n+\n+\t\t/* B.2 copy 2 64 bit or 4 32 bit mbuf point into rx_pkts */\n+\t\t_mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);\n+\n+#if defined(RTE_ARCH_X86_64)\n+\t\t/* B.1 load 2 64 bit mbuf points */\n+\t\tmbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos + 2]);\n+#endif\n+\n+\t\tdescs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));\n+\t\trte_compiler_barrier();\n+\t\t/* B.1 load 2 mbuf point */\n+\t\tdescs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));\n+\t\trte_compiler_barrier();\n+\t\tdescs[0] = _mm_loadu_si128((__m128i *)(rxdp));\n+\n+#if defined(RTE_ARCH_X86_64)\n+\t\t/* B.2 copy 2 mbuf point into rx_pkts */\n+\t\t_mm_storeu_si128((__m128i *)&rx_pkts[pos + 2], mbp2);\n+#endif\n+\n+\t\tif (split_packet) {\n+\t\t\trte_mbuf_prefetch_part2(rx_pkts[pos]);\n+\t\t\trte_mbuf_prefetch_part2(rx_pkts[pos + 1]);\n+\t\t\trte_mbuf_prefetch_part2(rx_pkts[pos + 2]);\n+\t\t\trte_mbuf_prefetch_part2(rx_pkts[pos + 3]);\n+\t\t}\n+\n+\t\t/* avoid compiler reorder optimization */\n+\t\trte_compiler_barrier();\n+\n+\t\t/* D.1 pkt 3,4 convert format from desc to pktmbuf */\n+\t\tpkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);\n+\t\tpkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);\n+\n+\t\t/* C.1 4=>2 filter staterr info only */\n+\t\tsterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]);\n+\t\t/* C.1 4=>2 filter staterr info only */\n+\t\tsterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]);\n+\n+\t\tflex_desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);\n+\n+\t\t/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */\n+\t\tpkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);\n+\t\tpkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);\n+\n+\t\t/* D.1 pkt 1,2 convert format from desc to pktmbuf */\n+\t\tpkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);\n+\t\tpkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);\n+\n+\t\t/* C.2 get 4 pkts staterr value */\n+\t\tstaterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);\n+\n+\t\t/* D.3 copy final 3,4 data to rx_pkts */\n+\t\t_mm_storeu_si128\n+\t\t\t((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,\n+\t\t\t pkt_mb4);\n+\t\t_mm_storeu_si128\n+\t\t\t((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,\n+\t\t\t pkt_mb3);\n+\n+\t\t/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */\n+\t\tpkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);\n+\t\tpkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);\n+\n+\t\t/* C* extract and record EOP bit */\n+\t\tif (split_packet) {\n+\t\t\t/* and with mask to extract bits, flipping 1-0 */\n+\t\t\t__m128i eop_bits = _mm_andnot_si128(staterr, eop_check);\n+\t\t\t/* the staterr values are not in order, as the count\n+\t\t\t * count of dd bits doesn't care. However, for end of\n+\t\t\t * packet tracking, we do care, so shuffle. This also\n+\t\t\t * compresses the 32-bit values to 8-bit\n+\t\t\t */\n+\t\t\teop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask);\n+\t\t\t/* store the resulting 32-bit value */\n+\t\t\t*(int *)split_packet = _mm_cvtsi128_si32(eop_bits);\n+\t\t\tsplit_packet += IAVF_VPMD_DESCS_PER_LOOP;\n+\t\t}\n+\n+\t\t/* C.3 calc available number of desc */\n+\t\tstaterr = _mm_and_si128(staterr, dd_check);\n+\t\tstaterr = _mm_packs_epi32(staterr, zero);\n+\n+\t\t/* D.3 copy final 1,2 data to rx_pkts */\n+\t\t_mm_storeu_si128\n+\t\t\t((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,\n+\t\t\t pkt_mb2);\n+\t\t_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,\n+\t\t\t\t pkt_mb1);\n+\t\tflex_desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);\n+\t\t/* C.4 calc avaialbe number of desc */\n+\t\tvar = __builtin_popcountll(_mm_cvtsi128_si64(staterr));\n+\t\tnb_pkts_recd += var;\n+\t\tif (likely(var != IAVF_VPMD_DESCS_PER_LOOP))\n+\t\t\tbreak;\n+\t}\n+\n+\t/* Update our internal tail pointer */\n+\trxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);\n+\trxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));\n+\trxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);\n+\n+\treturn nb_pkts_recd;\n+}\n+\n /* Notice:\n * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet\n * - nb_pkts > IAVF_VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST\n@@ -467,6 +827,18 @@ iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,\n \treturn _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);\n }\n \n+/* Notice:\n+ * - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet\n+ * - nb_pkts > IAVF_VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST\n+ * numbers of DD bits\n+ */\n+uint16_t\n+iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,\n+\t\t\t uint16_t nb_pkts)\n+{\n+\treturn _recv_raw_pkts_vec_flex_rxd(rx_queue, rx_pkts, nb_pkts, NULL);\n+}\n+\n /* vPMD receive routine that reassembles scattered packets\n * Notice:\n * - nb_pkts < IAVF_VPMD_DESCS_PER_LOOP, just return no packet\n@@ -508,6 +880,48 @@ iavf_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,\n \t\t&split_flags[i]);\n }\n \n+/* vPMD receive routine that reassembles scattered packets for flex RxD\n+ * Notice:\n+ * - nb_pkts < IAVF_VPMD_DESCS_PER_LOOP, just return no packet\n+ * - nb_pkts > VPMD_RX_MAX_BURST, only scan IAVF_VPMD_RX_MAX_BURST\n+ * numbers of DD bits\n+ */\n+uint16_t\n+iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,\n+\t\t\t\t struct rte_mbuf **rx_pkts,\n+\t\t\t\t uint16_t nb_pkts)\n+{\n+\tstruct iavf_rx_queue *rxq = rx_queue;\n+\tuint8_t split_flags[IAVF_VPMD_RX_MAX_BURST] = {0};\n+\tunsigned int i = 0;\n+\n+\t/* get some new buffers */\n+\tuint16_t nb_bufs = _recv_raw_pkts_vec_flex_rxd(rxq, rx_pkts, nb_pkts,\n+\t\t\t\t\t split_flags);\n+\tif (nb_bufs == 0)\n+\t\treturn 0;\n+\n+\t/* happy day case, full burst + no packets to be joined */\n+\tconst uint64_t *split_fl64 = (uint64_t *)split_flags;\n+\n+\tif (!rxq->pkt_first_seg &&\n+\t split_fl64[0] == 0 && split_fl64[1] == 0 &&\n+\t split_fl64[2] == 0 && split_fl64[3] == 0)\n+\t\treturn nb_bufs;\n+\n+\t/* reassemble any packets that need reassembly*/\n+\tif (!rxq->pkt_first_seg) {\n+\t\t/* find the first split flag, and only reassemble then*/\n+\t\twhile (i < nb_bufs && !split_flags[i])\n+\t\t\ti++;\n+\t\tif (i == nb_bufs)\n+\t\t\treturn nb_bufs;\n+\t\trxq->pkt_first_seg = rx_pkts[i];\n+\t}\n+\treturn i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,\n+\t\t&split_flags[i]);\n+}\n+\n static inline void\n vtx1(volatile struct iavf_tx_desc *txdp, struct rte_mbuf *pkt, uint64_t flags)\n {\n", "prefixes": [ "06/12" ] }