Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/52501/?format=api
{ "id": 52501, "url": "http://patches.dpdk.org/api/patches/52501/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20190409190630.31975-7-vivkong@ca.ibm.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20190409190630.31975-7-vivkong@ca.ibm.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20190409190630.31975-7-vivkong@ca.ibm.com", "date": "2019-04-09T19:06:24", "name": "[RFC,06/12] net/i40e: add support for s390x architecture", "commit_ref": null, "pull_url": null, "state": "changes-requested", "archived": true, "hash": "d8f57a104e532e6d74aafc3a1e7dbda960c46cfd", "submitter": { "id": 1273, "url": "http://patches.dpdk.org/api/people/1273/?format=api", "name": "Vivian Kong", "email": "vivkong@gmail.com" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20190409190630.31975-7-vivkong@ca.ibm.com/mbox/", "series": [ { "id": 4211, "url": "http://patches.dpdk.org/api/series/4211/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=4211", "date": "2019-04-09T19:06:18", "name": "introduce s390x architecture", "version": 1, "mbox": "http://patches.dpdk.org/series/4211/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/52501/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/52501/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 1D0625B34;\n\tTue, 9 Apr 2019 21:06:57 +0200 (CEST)", "from mail-qt1-f195.google.com (mail-qt1-f195.google.com\n\t[209.85.160.195]) by dpdk.org (Postfix) with ESMTP id 510445689\n\tfor <dev@dpdk.org>; Tue, 9 Apr 2019 21:06:42 +0200 (CEST)", "by mail-qt1-f195.google.com with SMTP id v20so21113561qtv.12\n\tfor <dev@dpdk.org>; Tue, 09 Apr 2019 12:06:42 -0700 (PDT)", "from csz25116.canlab.ibm.com ([199.246.40.57])\n\tby smtp.gmail.com with ESMTPSA id\n\tq23sm17934789qkc.16.2019.04.09.12.06.39 for <dev@dpdk.org>\n\t(version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);\n\tTue, 09 Apr 2019 12:06:39 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025;\n\th=from:to:subject:date:message-id:in-reply-to:references:reply-to;\n\tbh=MrwCmaa8+w4XyKn78I56FttoUZvb8kjswQcVZcFFDZQ=;\n\tb=l6Ktc9pnUVtyyOLFis/pkHQvA8ElRDK+HcFjQPCn1o1Zt88IaBQKjFIpI3qUDVPtYK\n\tyaGVbDbRmi450+mhEWryFzVENcAihLBIZfwR0y6sddCIEazQXJcGMTJAsyVcTdx4MNrk\n\tZvOPD82ekgJLbLV8Lkrc6rFI6bqzKmc+zexyAH530vS9MSlQLIu/or1PRbhqJbju8PUN\n\tc3rw26NePCZFzbyt4o1baC2BsTPaGspa6FHQHs7RpXTDiYemB+z/JwOKT0jYLJ0Uapey\n\tJBYSoF01sML93d5fujLzUGf9QjG2Cjp/xkQI3FeN1DBTl0NbCquFm3nOPACnt8F9BtZ3\n\tdKEw==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:from:to:subject:date:message-id:in-reply-to\n\t:references:reply-to;\n\tbh=MrwCmaa8+w4XyKn78I56FttoUZvb8kjswQcVZcFFDZQ=;\n\tb=lZ2bv8YsOxLucXYmFsGvxwvcfVt5VyBZHg93NGssciZ6KIc1o2y0Kj18DwAThxoanT\n\tijERp07jhky4OannDOa1n106O5vT7mXaRAdLJzskiAw355kvAqt9lNdqPbCUByUHA7xj\n\tx6kbaULT9wVXrdKSYr26EcTLddh3KDRzUqF9cigKJWcDwvTxBbinv3N8hpijXwS4f9kw\n\tNXphcjPjUWVVwQ1oO7jeyxZtl84dih2DzZ+6cgxK5Q94VkBvCCjQZQPshMzu2YPJKerD\n\t/33QvRirzCNU1FOzsvUVF2hlJBxD2clZG0Ptb5Gk83e0lzuxDTVVjI3uoeMp2ZCgv+jx\n\tDZlQ==", "X-Gm-Message-State": "APjAAAUlhIGionQuYmqP7RP3sYR3acJIuJIPdxadfoNg1TulW2rWDwqq\n\t/1I6WP5zYgs0ScWPTl+c/T2mrtzm6Mg=", "X-Google-Smtp-Source": "APXvYqyBpaUw4lKvYSulISN2RZazJNpQPgpGV13hpcyNl3CVLTzC5Yp3DezSvxrpUeYqX07dnd8Z2A==", "X-Received": "by 2002:ac8:8b9:: with SMTP id v54mr32965250qth.64.1554836800965;\n\tTue, 09 Apr 2019 12:06:40 -0700 (PDT)", "From": "Vivian Kong <vivkong@gmail.com>", "X-Google-Original-From": "Vivian Kong <vivkong@ca.ibm.com>", "To": "dev@dpdk.org", "Date": "Tue, 9 Apr 2019 15:06:24 -0400", "Message-Id": "<20190409190630.31975-7-vivkong@ca.ibm.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20190409190630.31975-1-vivkong@ca.ibm.com>", "References": "<20190409190630.31975-1-vivkong@ca.ibm.com>", "Subject": "[dpdk-dev] [RFC 06/12] net/i40e: add support for s390x architecture", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "Reply-To": "vivkong@ca.ibm.com", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Enable i40e and i40e vector support on s390x.\n\nSigned-off-by: Vivian Kong <vivkong@ca.ibm.com>\n---\n doc/guides/nics/features/i40e.ini | 1 +\n doc/guides/nics/features/i40e_vec.ini | 1 +\n drivers/net/i40e/Makefile | 2 +\n drivers/net/i40e/i40e_rxtx_vec_s390x.c | 631 +++++++++++++++++++++++++\n 4 files changed, 635 insertions(+)\n create mode 100644 drivers/net/i40e/i40e_rxtx_vec_s390x.c", "diff": "diff --git a/doc/guides/nics/features/i40e.ini b/doc/guides/nics/features/i40e.ini\nindex 16eab7f43..0c1acfc23 100644\n--- a/doc/guides/nics/features/i40e.ini\n+++ b/doc/guides/nics/features/i40e.ini\n@@ -55,3 +55,4 @@ x86-32 = Y\n x86-64 = Y\n ARMv8 = Y\n Power8 = Y\n+s390x = Y\ndiff --git a/doc/guides/nics/features/i40e_vec.ini b/doc/guides/nics/features/i40e_vec.ini\nindex c65e8b036..8c5062698 100644\n--- a/doc/guides/nics/features/i40e_vec.ini\n+++ b/doc/guides/nics/features/i40e_vec.ini\n@@ -43,3 +43,4 @@ x86-32 = Y\n x86-64 = Y\n ARMv8 = Y\n Power8 = Y\n+s390x = Y\ndiff --git a/drivers/net/i40e/Makefile b/drivers/net/i40e/Makefile\nindex 3f869a8d6..d13223cc9 100644\n--- a/drivers/net/i40e/Makefile\n+++ b/drivers/net/i40e/Makefile\n@@ -78,6 +78,8 @@ ifeq ($(CONFIG_RTE_ARCH_ARM64),y)\n SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_neon.c\n else ifeq ($(CONFIG_RTE_ARCH_PPC_64),y)\n SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_altivec.c\n+else ifeq ($(CONFIG_RTE_ARCH_S390X),y)\n+SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_s390x.c\n else\n SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_sse.c\n endif\ndiff --git a/drivers/net/i40e/i40e_rxtx_vec_s390x.c b/drivers/net/i40e/i40e_rxtx_vec_s390x.c\nnew file mode 100644\nindex 000000000..b35ab0678\n--- /dev/null\n+++ b/drivers/net/i40e/i40e_rxtx_vec_s390x.c\n@@ -0,0 +1,631 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.\n+ * (c) Copyright IBM Corp. 2017, 2019\n+ */\n+\n+#include <stdint.h>\n+#include <vecintrin.h>\n+#include <rte_ethdev_driver.h>\n+#include <rte_malloc.h>\n+\n+#include \"base/i40e_prototype.h\"\n+#include \"base/i40e_type.h\"\n+#include \"i40e_ethdev.h\"\n+#include \"i40e_rxtx.h\"\n+#include \"i40e_rxtx_vec_common.h\"\n+\n+#pragma GCC diagnostic ignored \"-Wcast-qual\"\n+\n+typedef unsigned long long vector_unsigned_long_long\n+\t__attribute__((vector_size(2 * sizeof(unsigned long long))));\n+typedef unsigned int vector_unsigned_int\n+\t__attribute__((vector_size(4 * sizeof(unsigned int))));\n+typedef unsigned short vector_unsigned_short\n+\t__attribute__((vector_size(8 * sizeof(unsigned short))));\n+typedef unsigned char vector_unsigned_char\n+\t__attribute__((vector_size(16 * sizeof(unsigned char))));\n+\n+\n+static inline void\n+i40e_rxq_rearm(struct i40e_rx_queue *rxq)\n+{\n+\tint i;\n+\tuint16_t rx_id;\n+\tvolatile union i40e_rx_desc *rxdp;\n+\n+\tstruct i40e_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];\n+\tstruct rte_mbuf *mb0, *mb1;\n+\n+\tvector_unsigned_long_long hdr_room = (vector_unsigned_long_long){\n+\t\t\t\t\t\tRTE_PKTMBUF_HEADROOM,\n+\t\t\t\t\t\tRTE_PKTMBUF_HEADROOM};\n+\tvector_unsigned_long_long dma_addr0, dma_addr1;\n+\n+\trxdp = rxq->rx_ring + rxq->rxrearm_start;\n+\n+\t/* Pull 'n' more MBUFs into the software ring */\n+\tif (rte_mempool_get_bulk(rxq->mp,\n+\t\t\t\t (void *)rxep,\n+\t\t\t\t RTE_I40E_RXQ_REARM_THRESH) < 0) {\n+\t\tif (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >=\n+\t\t rxq->nb_rx_desc) {\n+\t\t\tdma_addr0 = (vector_unsigned_long_long){};\n+\t\t\tfor (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) {\n+\t\t\t\trxep[i].mbuf = &rxq->fake_mbuf;\n+\t\t\t\tvec_xstd2(dma_addr0, 0,\n+\t\t\t\t\t(unsigned long long *)&rxdp[i].read);\n+\t\t\t}\n+\t\t}\n+\t\trte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=\n+\t\t\tRTE_I40E_RXQ_REARM_THRESH;\n+\t\treturn;\n+\t}\n+\n+\t/* Initialize the mbufs in vector, process 2 mbufs in one loop */\n+\tfor (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) {\n+\t\tvector_unsigned_long_long vaddr0, vaddr1;\n+\t\tuintptr_t p0, p1;\n+\n+\t\tmb0 = rxep[0].mbuf;\n+\t\tmb1 = rxep[1].mbuf;\n+\n+\t\t /* Flush mbuf with pkt template.\n+\t\t * Data to be rearmed is 6 bytes long.\n+\t\t * Though, RX will overwrite ol_flags that are coming next\n+\t\t * anyway. So overwrite whole 8 bytes with one load:\n+\t\t * 6 bytes of rearm_data plus first 2 bytes of ol_flags.\n+\t\t */\n+\t\tp0 = (uintptr_t)&mb0->rearm_data;\n+\t\t*(uint64_t *)p0 = rxq->mbuf_initializer;\n+\t\tp1 = (uintptr_t)&mb1->rearm_data;\n+\t\t*(uint64_t *)p1 = rxq->mbuf_initializer;\n+\n+\t\t/* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */\n+\t\tvaddr0 = vec_xld2(0, (unsigned long long *)&mb0->buf_addr);\n+\t\tvaddr1 = vec_xld2(0, (unsigned long long *)&mb1->buf_addr);\n+\n+\t\t/* convert pa to dma_addr hdr/data */\n+\t\tdma_addr0 = vec_mergel(vaddr0, vaddr0);\n+\t\tdma_addr1 = vec_mergel(vaddr1, vaddr1);\n+\n+\t\t/* add headroom to pa values */\n+\t\tdma_addr0 = dma_addr0 + hdr_room;\n+\t\tdma_addr1 = dma_addr1 + hdr_room;\n+\n+\t\t/* flush desc with pa dma_addr */\n+\t\tvec_xstd2(dma_addr0, 0, (unsigned long long *)&rxdp++->read);\n+\t\tvec_xstd2(dma_addr1, 0, (unsigned long long *)&rxdp++->read);\n+\t}\n+\n+\trxq->rxrearm_start += RTE_I40E_RXQ_REARM_THRESH;\n+\tif (rxq->rxrearm_start >= rxq->nb_rx_desc)\n+\t\trxq->rxrearm_start = 0;\n+\n+\trxq->rxrearm_nb -= RTE_I40E_RXQ_REARM_THRESH;\n+\n+\trx_id = (uint16_t)((rxq->rxrearm_start == 0) ?\n+\t\t\t (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));\n+\n+\t/* Update the tail pointer on the NIC */\n+\tI40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);\n+}\n+\n+static inline void\n+desc_to_olflags_v(vector_unsigned_long_long descs[4], struct rte_mbuf **rx_pkts)\n+{\n+\tvector_unsigned_int vlan0, vlan1, rss, l3_l4e;\n+\n+\t/* mask everything except RSS, flow director and VLAN flags\n+\t * bit2 is for VLAN tag, bit11 for flow director indication\n+\t * bit13:12 for RSS indication.\n+\t */\n+\tconst vector_unsigned_int rss_vlan_msk = (vector_unsigned_int){\n+\t\t\t(int32_t)0x1c03804, (int32_t)0x1c03804,\n+\t\t\t(int32_t)0x1c03804, (int32_t)0x1c03804};\n+\n+\t/* map rss and vlan type to rss hash and vlan flag */\n+\tconst vector_unsigned_char vlan_flags = (vector_unsigned_char){\n+\t\t\t0, 0, 0, 0,\n+\t\t\tPKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0, 0, 0,\n+\t\t\t0, 0, 0, 0,\n+\t\t\t0, 0, 0, 0};\n+\n+\tconst vector_unsigned_char rss_flags = (vector_unsigned_char){\n+\t\t\t0, PKT_RX_FDIR, 0, 0,\n+\t\t\t0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR,\n+\t\t\t0, 0, 0, 0,\n+\t\t\t0, 0, 0, 0};\n+\n+\tconst vector_unsigned_char l3_l4e_flags = (vector_unsigned_char){\n+\t\t\t0,\n+\t\t\tPKT_RX_IP_CKSUM_BAD,\n+\t\t\tPKT_RX_L4_CKSUM_BAD,\n+\t\t\tPKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD,\n+\t\t\tPKT_RX_EIP_CKSUM_BAD,\n+\t\t\tPKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD,\n+\t\t\tPKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD,\n+\t\t\tPKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD\n+\t\t\t\t\t | PKT_RX_IP_CKSUM_BAD,\n+\t\t\t0, 0, 0, 0, 0, 0, 0, 0};\n+\n+\tvlan0 = (vector_unsigned_int)vec_mergel(descs[0], descs[1]);\n+\tvlan1 = (vector_unsigned_int)vec_mergel(descs[2], descs[3]);\n+\tvlan0 = (vector_unsigned_int)vec_mergeh(vlan0, vlan1);\n+\n+\tvlan1 = vec_and(vlan0, rss_vlan_msk);\n+\tvlan0 = (vector_unsigned_int)vec_perm(vlan_flags,\n+\t\t\t\t\t(vector_unsigned_char){},\n+\t\t\t\t\t*(vector_unsigned_char *)&vlan1);\n+\n+\trss[0] = (uint32_t)vlan1[0] >> 11;\n+\trss[1] = (uint32_t)vlan1[1] >> 11;\n+\trss[2] = (uint32_t)vlan1[2] >> 11;\n+\trss[3] = (uint32_t)vlan1[3] >> 11;\n+\trss = (vector_unsigned_int)vec_perm(rss_flags, (vector_unsigned_char){},\n+\t\t\t\t\t*(vector_unsigned_char *)&rss);\n+\n+\tl3_l4e[0] = (uint32_t)vlan1[0] >> 22;\n+\tl3_l4e[1] = (uint32_t)vlan1[1] >> 22;\n+\tl3_l4e[2] = (uint32_t)vlan1[2] >> 22;\n+\tl3_l4e[3] = (uint32_t)vlan1[3] >> 22;\n+\n+\tl3_l4e = (vector_unsigned_int)vec_perm(l3_l4e_flags,\n+\t\t\t\t\t(vector_unsigned_char){},\n+\t\t\t\t\t*(vector_unsigned_char *)&l3_l4e);\n+\n+\tvlan0 = vec_or(vlan0, rss);\n+\tvlan0 = vec_or(vlan0, l3_l4e);\n+\n+\trx_pkts[0]->ol_flags = (uint64_t)vlan0[2];\n+\trx_pkts[1]->ol_flags = (uint64_t)vlan0[3];\n+\trx_pkts[2]->ol_flags = (uint64_t)vlan0[0];\n+\trx_pkts[3]->ol_flags = (uint64_t)vlan0[1];\n+}\n+\n+#define PKTLEN_SHIFT 10\n+\n+static inline void\n+desc_to_ptype_v(vector_unsigned_long_long descs[4], struct rte_mbuf **rx_pkts,\n+\t\tuint32_t *ptype_tbl)\n+{\n+\tvector_unsigned_long_long ptype0 = vec_mergel(descs[0], descs[1]);\n+\tvector_unsigned_long_long ptype1 = vec_mergel(descs[2], descs[3]);\n+\n+\tptype0[0] = ptype0[0] >> 30;\n+\tptype0[1] = ptype0[1] >> 30;\n+\n+\tptype1[0] = ptype1[0] >> 30;\n+\tptype1[1] = ptype1[1] >> 30;\n+\n+\trx_pkts[0]->packet_type =\n+\t\tptype_tbl[(*(vector_unsigned_char *)&ptype0)[0]];\n+\trx_pkts[1]->packet_type =\n+\t\tptype_tbl[(*(vector_unsigned_char *)&ptype0)[8]];\n+\trx_pkts[2]->packet_type =\n+\t\tptype_tbl[(*(vector_unsigned_char *)&ptype1)[0]];\n+\trx_pkts[3]->packet_type =\n+\t\tptype_tbl[(*(vector_unsigned_char *)&ptype1)[8]];\n+}\n+\n+ /* Notice:\n+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet\n+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST\n+ * numbers of DD bits\n+ */\n+static inline uint16_t\n+_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,\n+\t\t uint16_t nb_pkts, uint8_t *split_packet)\n+{\n+\tvolatile union i40e_rx_desc *rxdp;\n+\tstruct i40e_rx_entry *sw_ring;\n+\tuint16_t nb_pkts_recd;\n+\tint pos;\n+\tuint64_t var;\n+\tvector_unsigned_char shuf_msk;\n+\tuint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;\n+\n+\tvector_unsigned_short crc_adjust = (vector_unsigned_short){\n+\t\t0, 0, /* ignore pkt_type field */\n+\t\trxq->crc_len, /* sub crc on pkt_len */\n+\t\t0, /* ignore high-16bits of pkt_len */\n+\t\trxq->crc_len, /* sub crc on data_len */\n+\t\t0, 0, 0 /* ignore non-length fields */\n+\t\t};\n+\tvector_unsigned_long_long dd_check, eop_check;\n+\n+\t/* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */\n+\tnb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST);\n+\n+\t/* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */\n+\tnb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP);\n+\n+\t/* Just the act of getting into the function from the application is\n+\t * going to cost about 7 cycles\n+\t */\n+\trxdp = rxq->rx_ring + rxq->rx_tail;\n+\n+\trte_prefetch0(rxdp);\n+\n+\t/* See if we need to rearm the RX queue - gives the prefetch a bit\n+\t * of time to act\n+\t */\n+\tif (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH)\n+\t\ti40e_rxq_rearm(rxq);\n+\n+\t/* Before we start moving massive data around, check to see if\n+\t * there is actually a packet available\n+\t */\n+\tif (!(rxdp->wb.qword1.status_error_len &\n+\t\t\trte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT)))\n+\t\treturn 0;\n+\n+\t/* 4 packets DD mask */\n+\tdd_check = (vector_unsigned_long_long){0x0000000100000001ULL,\n+\t\t\t\t\t 0x0000000100000001ULL};\n+\n+\t/* 4 packets EOP mask */\n+\teop_check = (vector_unsigned_long_long){0x0000000200000002ULL,\n+\t\t\t\t\t 0x0000000200000002ULL};\n+\n+\t/* mask to shuffle from desc. to mbuf */\n+\tshuf_msk = (vector_unsigned_char){\n+\t\t0xFF, 0xFF, /* pkt_type set as unknown */\n+\t\t0xFF, 0xFF, /* pkt_type set as unknown */\n+\t\t14, 15, /* octet 15~14, low 16 bits pkt_len */\n+\t\t0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */\n+\t\t14, 15, /* octet 15~14, 16 bits data_len */\n+\t\t2, 3, /* octet 2~3, low 16 bits vlan_macip */\n+\t\t4, 5, 6, 7 /* octet 4~7, 32bits rss */\n+\t\t};\n+\n+\t/* Cache is empty -> need to scan the buffer rings, but first move\n+\t * the next 'n' mbufs into the cache\n+\t */\n+\tsw_ring = &rxq->sw_ring[rxq->rx_tail];\n+\n+\t/* A. load 4 packet in one loop\n+\t * [A*. mask out 4 unused dirty field in desc]\n+\t * B. copy 4 mbuf point from swring to rx_pkts\n+\t * C. calc the number of DD bits among the 4 packets\n+\t * [C*. extract the end-of-packet bit, if requested]\n+\t * D. fill info. from desc to mbuf\n+\t */\n+\n+\tfor (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;\n+\t\t\tpos += RTE_I40E_DESCS_PER_LOOP,\n+\t\t\trxdp += RTE_I40E_DESCS_PER_LOOP) {\n+\t\tvector_unsigned_long_long descs[RTE_I40E_DESCS_PER_LOOP];\n+\t\tvector_unsigned_char pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;\n+\t\tvector_unsigned_short staterr, sterr_tmp1, sterr_tmp2;\n+\t\tvector_unsigned_long_long mbp1, mbp2; /* two mbuf pointer\n+\t\t\t\t\t\t\t* in one XMM reg.\n+\t\t\t\t\t\t\t*/\n+\n+\t\t/* B.1 load 1 mbuf point */\n+\t\tmbp1 = *(vector_unsigned_long_long *)&sw_ring[pos];\n+\t\t/* Read desc statuses backwards to avoid race condition */\n+\t\t/* A.1 load 4 pkts desc */\n+\t\tdescs[3] = *(vector_unsigned_long_long *)(rxdp + 3);\n+\t\trte_compiler_barrier();\n+\n+\t\t/* B.2 copy 2 mbuf point into rx_pkts */\n+\t\t*(vector_unsigned_long_long *)&rx_pkts[pos] = mbp1;\n+\n+\t\t/* B.1 load 1 mbuf point */\n+\t\tmbp2 = *(vector_unsigned_long_long *)&sw_ring[pos + 2];\n+\n+\t\tdescs[2] = *(vector_unsigned_long_long *)(rxdp + 2);\n+\t\trte_compiler_barrier();\n+\t\t/* B.1 load 2 mbuf point */\n+\t\tdescs[1] = *(vector_unsigned_long_long *)(rxdp + 1);\n+\t\trte_compiler_barrier();\n+\t\tdescs[0] = *(vector_unsigned_long_long *)(rxdp);\n+\n+\t\t/* B.2 copy 2 mbuf point into rx_pkts */\n+\t\t*(vector_unsigned_long_long *)&rx_pkts[pos + 2] = mbp2;\n+\n+\t\tif (split_packet) {\n+\t\t\trte_mbuf_prefetch_part2(rx_pkts[pos]);\n+\t\t\trte_mbuf_prefetch_part2(rx_pkts[pos + 1]);\n+\t\t\trte_mbuf_prefetch_part2(rx_pkts[pos + 2]);\n+\t\t\trte_mbuf_prefetch_part2(rx_pkts[pos + 3]);\n+\t\t}\n+\n+\t\t/* avoid compiler reorder optimization */\n+\t\trte_compiler_barrier();\n+\n+\t\t/* pkt 3,4 shift the pktlen field to be 16-bit aligned*/\n+\t\tvector_unsigned_int len3_temp = vec_xld2(0,\n+\t\t\t\t(unsigned int *)&descs[3]);\n+\t\tlen3_temp[3] = len3_temp[3] << PKTLEN_SHIFT;\n+\t\tconst vector_unsigned_int len3 = len3_temp;\n+\n+\t\tvector_unsigned_int len2_temp = vec_xld2(0,\n+\t\t\t\t(unsigned int *)&descs[2]);\n+\t\tlen2_temp[3] = len2_temp[3] << PKTLEN_SHIFT;\n+\t\tconst vector_unsigned_int len2 = len2_temp;\n+\n+\t\t/* merge the now-aligned packet length fields back in */\n+\t\tdescs[3] = (vector_unsigned_long_long)len3;\n+\t\tdescs[2] = (vector_unsigned_long_long)len2;\n+\n+\t\t/* D.1 pkt 3,4 convert format from desc to pktmbuf */\n+\t\tpkt_mb4 = vec_perm((vector_unsigned_char)descs[3],\n+\t\t\t\t (vector_unsigned_char){}, shuf_msk);\n+\t\tpkt_mb3 = vec_perm((vector_unsigned_char)descs[2],\n+\t\t\t\t (vector_unsigned_char){}, shuf_msk);\n+\n+\t\t/* C.1 4=>2 filter staterr info only */\n+\t\tsterr_tmp2 = vec_mergel((vector_unsigned_short)descs[3],\n+\t\t\t\t\t(vector_unsigned_short)descs[2]);\n+\t\t/* C.1 4=>2 filter staterr info only */\n+\t\tsterr_tmp1 = vec_mergel((vector_unsigned_short)descs[1],\n+\t\t\t\t\t(vector_unsigned_short)descs[0]);\n+\t\t/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */\n+\t\tpkt_mb4 = (vector_unsigned_char)((vector_unsigned_short)pkt_mb4\n+\t\t\t\t- crc_adjust);\n+\t\tpkt_mb3 = (vector_unsigned_char)((vector_unsigned_short)pkt_mb3\n+\t\t\t\t- crc_adjust);\n+\n+\t\t/* pkt 1,2 shift the pktlen field to be 16-bit aligned*/\n+\t\tconst vector_unsigned_int len1 =\n+\t\t\tvec_sll(vec_xld2(0, (unsigned int *)&descs[1]),\n+\t\t\t(vector_unsigned_int){0, 0, 0, PKTLEN_SHIFT});\n+\t\tconst vector_unsigned_int len0 =\n+\t\t\tvec_sll(vec_xld2(0, (unsigned int *)&descs[0]),\n+\t\t\t(vector_unsigned_int){0, 0, 0, PKTLEN_SHIFT});\n+\n+\t\t/* merge the now-aligned packet length fields back in */\n+\t\tdescs[1] = (vector_unsigned_long_long)len1;\n+\t\tdescs[0] = (vector_unsigned_long_long)len0;\n+\n+\t\t/* D.1 pkt 1,2 convert format from desc to pktmbuf */\n+\t\tpkt_mb2 = vec_perm((vector_unsigned_char)descs[1],\n+\t\t\t\t (vector_unsigned_char){}, shuf_msk);\n+\t\tpkt_mb1 = vec_perm((vector_unsigned_char)descs[0],\n+\t\t\t\t (vector_unsigned_char){}, shuf_msk);\n+\n+\t\t/* C.2 get 4 pkts staterr value */\n+\t\tstaterr = (vector_unsigned_short)vec_mergeh(sterr_tmp1,\n+\t\t\t\tsterr_tmp2);\n+\n+\t\t/* D.3 copy final 3,4 data to rx_pkts */\n+\t\tvec_xstd2(pkt_mb4, 0, (unsigned char *)&rx_pkts[pos + 3]\n+\t\t\t->rx_descriptor_fields1);\n+\t\tvec_xstd2(pkt_mb3, 0, (unsigned char *)&rx_pkts[pos + 2]\n+\t\t\t->rx_descriptor_fields1);\n+\n+\t\t/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */\n+\t\tpkt_mb2 = (vector_unsigned_char)((vector_unsigned_short)pkt_mb2\n+\t\t\t\t- crc_adjust);\n+\t\tpkt_mb1 = (vector_unsigned_char)((vector_unsigned_short)pkt_mb1\n+\t\t\t\t- crc_adjust);\n+\n+\t\t/* C* extract and record EOP bit */\n+\t\tif (split_packet) {\n+\t\t\tvector_unsigned_char eop_shuf_mask =\n+\t\t\t\t(vector_unsigned_char){\n+\t\t\t\t\t0xFF, 0xFF, 0xFF, 0xFF,\n+\t\t\t\t\t0xFF, 0xFF, 0xFF, 0xFF,\n+\t\t\t\t\t0xFF, 0xFF, 0xFF, 0xFF,\n+\t\t\t\t\t0x04, 0x0C, 0x00, 0x08\n+\t\t\t\t};\n+\n+\t\t\t/* and with mask to extract bits, flipping 1-0 */\n+\t\t\tvector_unsigned_char eop_bits =\n+\t\t\t\tvec_and((vector_unsigned_char)vec_nor(staterr,\n+\t\t\t\tstaterr), (vector_unsigned_char)eop_check);\n+\t\t\t/* the staterr values are not in order, as the count\n+\t\t\t * count of dd bits doesn't care. However, for end of\n+\t\t\t * packet tracking, we do care, so shuffle. This also\n+\t\t\t * compresses the 32-bit values to 8-bit\n+\t\t\t */\n+\t\t\teop_bits = vec_perm(eop_bits, (vector_unsigned_char){},\n+\t\t\t\t\t eop_shuf_mask);\n+\t\t\t/* store the resulting 32-bit value */\n+\t\t\t*split_packet = (vec_xld2(0,\n+\t\t\t\t\t (unsigned int *)&eop_bits))[0];\n+\t\t\tsplit_packet += RTE_I40E_DESCS_PER_LOOP;\n+\n+\t\t\t/* zero-out next pointers */\n+\t\t\trx_pkts[pos]->next = NULL;\n+\t\t\trx_pkts[pos + 1]->next = NULL;\n+\t\t\trx_pkts[pos + 2]->next = NULL;\n+\t\t\trx_pkts[pos + 3]->next = NULL;\n+\t\t}\n+\n+\t\t/* C.3 calc available number of desc */\n+\t\tstaterr = vec_and(staterr, (vector_unsigned_short)dd_check);\n+\n+\t\t/* D.3 copy final 1,2 data to rx_pkts */\n+\t\tvec_xstd2(pkt_mb2, 0, (unsigned char *)&rx_pkts[pos + 1]\n+\t\t\t->rx_descriptor_fields1);\n+\t\tvec_xstd2(pkt_mb1, 0, (unsigned char *)&rx_pkts[pos]\n+\t\t\t->rx_descriptor_fields1);\n+\n+\t\tdesc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);\n+\t\tdesc_to_olflags_v(descs, &rx_pkts[pos]);\n+\n+\t\t/* C.4 calc avaialbe number of desc */\n+\t\tvar = __builtin_popcountll((vec_xld2(0,\n+\t\t\t(unsigned long long *)&staterr)[0]));\n+\t\tnb_pkts_recd += var;\n+\t\tif (likely(var != RTE_I40E_DESCS_PER_LOOP))\n+\t\t\tbreak;\n+\t}\n+\n+\t/* Update our internal tail pointer */\n+\trxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);\n+\trxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));\n+\trxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);\n+\n+\treturn nb_pkts_recd;\n+}\n+\n+ /* Notice:\n+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet\n+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST\n+ * numbers of DD bits\n+ */\n+uint16_t\n+i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,\n+\t\t uint16_t nb_pkts)\n+{\n+\treturn _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);\n+}\n+\n+ /* vPMD receive routine that reassembles scattered packets\n+ * Notice:\n+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet\n+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST\n+ * numbers of DD bits\n+ */\n+uint16_t\n+i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,\n+\t\t\t uint16_t nb_pkts)\n+{\n+\tstruct i40e_rx_queue *rxq = rx_queue;\n+\tuint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0};\n+\n+\t/* get some new buffers */\n+\tuint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts,\n+\t\t\tsplit_flags);\n+\tif (nb_bufs == 0)\n+\t\treturn 0;\n+\n+\t/* happy day case, full burst + no packets to be joined */\n+\tconst uint64_t *split_fl64 = (uint64_t *)split_flags;\n+\n+\tif (rxq->pkt_first_seg == NULL &&\n+\t split_fl64[0] == 0 && split_fl64[1] == 0 &&\n+\t split_fl64[2] == 0 && split_fl64[3] == 0)\n+\t\treturn nb_bufs;\n+\n+\t/* reassemble any packets that need reassembly*/\n+\tunsigned int i = 0;\n+\n+\tif (!rxq->pkt_first_seg) {\n+\t\t/* find the first split flag, and only reassemble then*/\n+\t\twhile (i < nb_bufs && !split_flags[i])\n+\t\t\ti++;\n+\t\tif (i == nb_bufs)\n+\t\t\treturn nb_bufs;\n+\t}\n+\treturn i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,\n+\t\t&split_flags[i]);\n+}\n+\n+static inline void\n+vtx1(volatile struct i40e_tx_desc *txdp,\n+\tstruct rte_mbuf *pkt, uint64_t flags)\n+{\n+\tuint64_t high_qw = (I40E_TX_DESC_DTYPE_DATA |\n+\t\t((uint64_t)flags << I40E_TXD_QW1_CMD_SHIFT) |\n+\t\t((uint64_t)pkt->data_len << I40E_TXD_QW1_TX_BUF_SZ_SHIFT));\n+\n+\tvector_unsigned_long_long descriptor = (vector_unsigned_long_long){\n+\t\tpkt->buf_iova + pkt->data_off, high_qw};\n+\t*(vector_unsigned_long_long *)txdp = descriptor;\n+}\n+\n+static inline void\n+vtx(volatile struct i40e_tx_desc *txdp,\n+\tstruct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt)\n+\t\tvtx1(txdp, *pkt, flags);\n+}\n+\n+uint16_t\n+i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,\n+\t\t\t uint16_t nb_pkts)\n+{\n+\tstruct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;\n+\tvolatile struct i40e_tx_desc *txdp;\n+\tstruct i40e_tx_entry *txep;\n+\tuint16_t n, nb_commit, tx_id;\n+\tuint64_t flags = I40E_TD_CMD;\n+\tuint64_t rs = I40E_TX_DESC_CMD_RS | I40E_TD_CMD;\n+\tint i;\n+\n+\t/* cross rx_thresh boundary is not allowed */\n+\tnb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh);\n+\n+\tif (txq->nb_tx_free < txq->tx_free_thresh)\n+\t\ti40e_tx_free_bufs(txq);\n+\n+\tnb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);\n+\tnb_commit = nb_pkts;\n+\tif (unlikely(nb_pkts == 0))\n+\t\treturn 0;\n+\n+\ttx_id = txq->tx_tail;\n+\ttxdp = &txq->tx_ring[tx_id];\n+\ttxep = &txq->sw_ring[tx_id];\n+\n+\ttxq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);\n+\n+\tn = (uint16_t)(txq->nb_tx_desc - tx_id);\n+\tif (nb_commit >= n) {\n+\t\ttx_backlog_entry(txep, tx_pkts, n);\n+\n+\t\tfor (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp)\n+\t\t\tvtx1(txdp, *tx_pkts, flags);\n+\n+\t\tvtx1(txdp, *tx_pkts++, rs);\n+\n+\t\tnb_commit = (uint16_t)(nb_commit - n);\n+\n+\t\ttx_id = 0;\n+\t\ttxq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);\n+\n+\t\t/* avoid reach the end of ring */\n+\t\ttxdp = &txq->tx_ring[tx_id];\n+\t\ttxep = &txq->sw_ring[tx_id];\n+\t}\n+\n+\ttx_backlog_entry(txep, tx_pkts, nb_commit);\n+\n+\tvtx(txdp, tx_pkts, nb_commit, flags);\n+\n+\ttx_id = (uint16_t)(tx_id + nb_commit);\n+\tif (tx_id > txq->tx_next_rs) {\n+\t\ttxq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=\n+\t\t\trte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<\n+\t\t\t\t\t\tI40E_TXD_QW1_CMD_SHIFT);\n+\t\ttxq->tx_next_rs =\n+\t\t\t(uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);\n+\t}\n+\n+\ttxq->tx_tail = tx_id;\n+\n+\tI40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);\n+\n+\treturn nb_pkts;\n+}\n+\n+void __attribute__((cold))\n+i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)\n+{\n+\t_i40e_rx_queue_release_mbufs_vec(rxq);\n+}\n+\n+int __attribute__((cold))\n+i40e_rxq_vec_setup(struct i40e_rx_queue *rxq)\n+{\n+\treturn i40e_rxq_vec_setup_default(rxq);\n+}\n+\n+int __attribute__((cold))\n+i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused * txq)\n+{\n+\treturn 0;\n+}\n+\n+int __attribute__((cold))\n+i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)\n+{\n+\treturn i40e_rx_vec_dev_conf_condition_check_default(dev);\n+}\n", "prefixes": [ "RFC", "06/12" ] }