Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 118778,
    "url": "https://patches.dpdk.org/api/patches/118778/?format=api",
    "web_url": "https://patches.dpdk.org/project/dpdk/patch/20221020103656.1068036-9-junfeng.guo@intel.com/",
    "project": {
        "id": 1,
        "url": "https://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20221020103656.1068036-9-junfeng.guo@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20221020103656.1068036-9-junfeng.guo@intel.com",
    "date": "2022-10-20T10:36:56",
    "name": "[v6,8/8] net/gve: add support for Rx/Tx",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "45096b85e1b4e69af8582684ad6142c99b9675e1",
    "submitter": {
        "id": 1785,
        "url": "https://patches.dpdk.org/api/people/1785/?format=api",
        "name": "Junfeng Guo",
        "email": "junfeng.guo@intel.com"
    },
    "delegate": {
        "id": 319,
        "url": "https://patches.dpdk.org/api/users/319/?format=api",
        "username": "fyigit",
        "first_name": "Ferruh",
        "last_name": "Yigit",
        "email": "ferruh.yigit@amd.com"
    },
    "mbox": "https://patches.dpdk.org/project/dpdk/patch/20221020103656.1068036-9-junfeng.guo@intel.com/mbox/",
    "series": [
        {
            "id": 25340,
            "url": "https://patches.dpdk.org/api/series/25340/?format=api",
            "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=25340",
            "date": "2022-10-20T10:36:48",
            "name": "introduce GVE PMD",
            "version": 6,
            "mbox": "https://patches.dpdk.org/series/25340/mbox/"
        }
    ],
    "comments": "https://patches.dpdk.org/api/patches/118778/comments/",
    "check": "success",
    "checks": "https://patches.dpdk.org/api/patches/118778/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 96D68A0581;\n\tThu, 20 Oct 2022 12:39:23 +0200 (CEST)",
            "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 3661642D54;\n\tThu, 20 Oct 2022 12:39:01 +0200 (CEST)",
            "from mga03.intel.com (mga03.intel.com [134.134.136.65])\n by mails.dpdk.org (Postfix) with ESMTP id 3FC6442D43\n for <dev@dpdk.org>; Thu, 20 Oct 2022 12:38:56 +0200 (CEST)",
            "from fmsmga004.fm.intel.com ([10.253.24.48])\n by orsmga103.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 20 Oct 2022 03:38:55 -0700",
            "from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104])\n by fmsmga004.fm.intel.com with ESMTP; 20 Oct 2022 03:38:52 -0700"
        ],
        "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1666262336; x=1697798336;\n h=from:to:cc:subject:date:message-id:in-reply-to:\n references:mime-version:content-transfer-encoding;\n bh=GuGR4rMypgwHGFMrii+lg1vIZi6r0M2Lb5VunNuX7yc=;\n b=gNS44ZEWm99fN8Db3PR9Ixdi19MgfBE/7WmVBfKX82Hn+9Eepmypcy7/\n 5bQh0VhfiNFizolbeNoWt3AcQ8pncCppF24S59pwtF1dUeAXfDdTheu3J\n q0eU2jFXiWSxenHPpsiIOS9GZofZniBmYe9/gWBQDN2Kwos0YxvhAXox8\n fMINFsnQBZL0P9Q1JfmXHQH3mMJ3XKLnUXmBsiAdYKRINlt0L++wFj0PP\n tqX+inQlmVzCZC9Tk3/gr+C4wSN/hmwegQuUE+CE7SVnJtW9ZXLI+8I34\n 4G4J27X6NQpuwqGB/4cEYGz0o1EYyLo/m8Cr/VV6UUCKXrxN9epa5wLMu A==;",
        "X-IronPort-AV": [
            "E=McAfee;i=\"6500,9779,10505\"; a=\"308354947\"",
            "E=Sophos;i=\"5.95,198,1661842800\"; d=\"scan'208\";a=\"308354947\"",
            "E=McAfee;i=\"6500,9779,10505\"; a=\"698582935\"",
            "E=Sophos;i=\"5.95,198,1661842800\"; d=\"scan'208\";a=\"698582935\""
        ],
        "X-ExtLoop1": "1",
        "From": "Junfeng Guo <junfeng.guo@intel.com>",
        "To": "qi.z.zhang@intel.com, jingjing.wu@intel.com, ferruh.yigit@xilinx.com,\n beilei.xing@intel.com",
        "Cc": "dev@dpdk.org, xiaoyun.li@intel.com, awogbemila@google.com,\n bruce.richardson@intel.com, hemant.agrawal@nxp.com,\n stephen@networkplumber.org, chenbo.xia@intel.com, helin.zhang@intel.com,\n Junfeng Guo <junfeng.guo@intel.com>",
        "Subject": "[PATCH v6 8/8] net/gve: add support for Rx/Tx",
        "Date": "Thu, 20 Oct 2022 18:36:56 +0800",
        "Message-Id": "<20221020103656.1068036-9-junfeng.guo@intel.com>",
        "X-Mailer": "git-send-email 2.34.1",
        "In-Reply-To": "<20221020103656.1068036-1-junfeng.guo@intel.com>",
        "References": "<20221010101757.878317-2-junfeng.guo@intel.com>\n <20221020103656.1068036-1-junfeng.guo@intel.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org"
    },
    "content": "Add Rx/Tx of GQI_QPL queue format and GQI_RDA queue format.\n\nSigned-off-by: Xiaoyun Li <xiaoyun.li@intel.com>\nSigned-off-by: Junfeng Guo <junfeng.guo@intel.com>\n---\n doc/guides/nics/features/gve.ini |   2 +\n doc/guides/nics/gve.rst          |   4 +\n drivers/net/gve/gve_ethdev.c     |  15 +-\n drivers/net/gve/gve_ethdev.h     |  18 ++\n drivers/net/gve/gve_rx.c         | 142 ++++++++++\n drivers/net/gve/gve_tx.c         | 455 +++++++++++++++++++++++++++++++\n 6 files changed, 635 insertions(+), 1 deletion(-)",
    "diff": "diff --git a/doc/guides/nics/features/gve.ini b/doc/guides/nics/features/gve.ini\nindex 986df7f94a..cdc46b08a3 100644\n--- a/doc/guides/nics/features/gve.ini\n+++ b/doc/guides/nics/features/gve.ini\n@@ -7,7 +7,9 @@\n Speed capabilities   = Y\n Link status          = Y\n MTU update           = Y\n+TSO                  = Y\n RSS hash             = Y\n+L4 checksum offload  = Y\n Linux                = Y\n x86-32               = Y\n x86-64               = Y\ndiff --git a/doc/guides/nics/gve.rst b/doc/guides/nics/gve.rst\nindex 8c09a5a7fa..1042852fd6 100644\n--- a/doc/guides/nics/gve.rst\n+++ b/doc/guides/nics/gve.rst\n@@ -62,8 +62,12 @@ In this release, the GVE PMD provides the basic functionality of packet\n reception and transmission.\n Supported features of the GVE PMD are:\n \n+- Multiple queues for TX and RX\n - Receiver Side Scaling (RSS)\n+- TSO offload\n - Link state information\n+- TX multi-segments (Scatter TX)\n+- Tx UDP/TCP/SCTP Checksum\n \n Currently, only GQI_QPL and GQI_RDA queue format are supported in PMD.\n Jumbo Frame is not supported in PMD for now. It'll be added in the future\ndiff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c\nindex 52c335bc16..91f930957d 100644\n--- a/drivers/net/gve/gve_ethdev.c\n+++ b/drivers/net/gve/gve_ethdev.c\n@@ -284,7 +284,13 @@ gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)\n \tdev_info->min_mtu = RTE_ETHER_MIN_MTU;\n \n \tdev_info->rx_offload_capa = 0;\n-\tdev_info->tx_offload_capa = 0;\n+\tdev_info->tx_offload_capa =\n+\t\tRTE_ETH_TX_OFFLOAD_MULTI_SEGS\t|\n+\t\tRTE_ETH_TX_OFFLOAD_IPV4_CKSUM\t|\n+\t\tRTE_ETH_TX_OFFLOAD_UDP_CKSUM\t|\n+\t\tRTE_ETH_TX_OFFLOAD_TCP_CKSUM\t|\n+\t\tRTE_ETH_TX_OFFLOAD_SCTP_CKSUM\t|\n+\t\tRTE_ETH_TX_OFFLOAD_TCP_TSO;\n \n \tif (priv->queue_format == GVE_DQO_RDA_FORMAT)\n \t\tdev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO;\n@@ -633,6 +639,13 @@ gve_dev_init(struct rte_eth_dev *eth_dev)\n \tif (err)\n \t\treturn err;\n \n+\tif (gve_is_gqi(priv)) {\n+\t\teth_dev->rx_pkt_burst = gve_rx_burst;\n+\t\teth_dev->tx_pkt_burst = gve_tx_burst;\n+\t} else {\n+\t\tPMD_DRV_LOG(ERR, \"DQO_RDA is not implemented and will be added in the future\");\n+\t}\n+\n \teth_dev->data->mac_addrs = rte_zmalloc(\"gve_mac\", sizeof(struct rte_ether_addr), 0);\n \tif (!eth_dev->data->mac_addrs) {\n \t\tPMD_DRV_LOG(ERR, \"Failed to allocate memory to store mac address\");\ndiff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h\nindex 20fe57781e..266b831a01 100644\n--- a/drivers/net/gve/gve_ethdev.h\n+++ b/drivers/net/gve/gve_ethdev.h\n@@ -34,6 +34,18 @@ union gve_tx_desc {\n \tstruct gve_tx_seg_desc seg; /* subsequent descs for a packet */\n };\n \n+/* Offload features */\n+union gve_tx_offload {\n+\tuint64_t data;\n+\tstruct {\n+\t\tuint64_t l2_len:7; /* L2 (MAC) Header Length. */\n+\t\tuint64_t l3_len:9; /* L3 (IP) Header Length. */\n+\t\tuint64_t l4_len:8; /* L4 Header Length. */\n+\t\tuint64_t tso_segsz:16; /* TCP TSO segment size */\n+\t\t/* uint64_t unused : 24; */\n+\t};\n+};\n+\n struct gve_tx_iovec {\n \tuint32_t iov_base; /* offset in fifo */\n \tuint32_t iov_len;\n@@ -274,4 +286,10 @@ gve_stop_tx_queues(struct rte_eth_dev *dev);\n void\n gve_stop_rx_queues(struct rte_eth_dev *dev);\n \n+uint16_t\n+gve_rx_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);\n+\n+uint16_t\n+gve_tx_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);\n+\n #endif /* _GVE_ETHDEV_H_ */\ndiff --git a/drivers/net/gve/gve_rx.c b/drivers/net/gve/gve_rx.c\nindex e64a461253..ea397d68fa 100644\n--- a/drivers/net/gve/gve_rx.c\n+++ b/drivers/net/gve/gve_rx.c\n@@ -5,6 +5,148 @@\n #include \"gve_ethdev.h\"\n #include \"base/gve_adminq.h\"\n \n+static inline void\n+gve_rx_refill(struct gve_rx_queue *rxq)\n+{\n+\tuint16_t mask = rxq->nb_rx_desc - 1;\n+\tuint16_t idx = rxq->next_avail & mask;\n+\tuint32_t next_avail = rxq->next_avail;\n+\tuint16_t nb_alloc, i;\n+\tstruct rte_mbuf *nmb;\n+\tint diag;\n+\n+\t/* wrap around */\n+\tnb_alloc = rxq->nb_rx_desc - idx;\n+\tif (nb_alloc <= rxq->nb_avail) {\n+\t\tdiag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc);\n+\t\tif (diag < 0) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rte_pktmbuf_alloc(rxq->mpool);\n+\t\t\t\tif (!nmb)\n+\t\t\t\t\tbreak;\n+\t\t\t\trxq->sw_ring[idx + i] = nmb;\n+\t\t\t}\n+\t\t\tif (i != nb_alloc)\n+\t\t\t\tnb_alloc = i;\n+\t\t}\n+\t\trxq->nb_avail -= nb_alloc;\n+\t\tnext_avail += nb_alloc;\n+\n+\t\t/* queue page list mode doesn't need real refill. */\n+\t\tif (rxq->is_gqi_qpl) {\n+\t\t\tidx += nb_alloc;\n+\t\t} else {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rxq->sw_ring[idx];\n+\t\t\t\trxq->rx_data_ring[idx].addr =\n+\t\t\t\t\trte_cpu_to_be_64(rte_mbuf_data_iova(nmb));\n+\t\t\t\tidx++;\n+\t\t\t}\n+\t\t}\n+\t\tif (idx == rxq->nb_rx_desc)\n+\t\t\tidx = 0;\n+\t}\n+\n+\tif (rxq->nb_avail > 0) {\n+\t\tnb_alloc = rxq->nb_avail;\n+\t\tif (rxq->nb_rx_desc < idx + rxq->nb_avail)\n+\t\t\tnb_alloc = rxq->nb_rx_desc - idx;\n+\t\tdiag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc);\n+\t\tif (diag < 0) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rte_pktmbuf_alloc(rxq->mpool);\n+\t\t\t\tif (!nmb)\n+\t\t\t\t\tbreak;\n+\t\t\t\trxq->sw_ring[idx + i] = nmb;\n+\t\t\t}\n+\t\t\tnb_alloc = i;\n+\t\t}\n+\t\trxq->nb_avail -= nb_alloc;\n+\t\tnext_avail += nb_alloc;\n+\n+\t\tif (!rxq->is_gqi_qpl) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rxq->sw_ring[idx];\n+\t\t\t\trxq->rx_data_ring[idx].addr =\n+\t\t\t\t\trte_cpu_to_be_64(rte_mbuf_data_iova(nmb));\n+\t\t\t\tidx++;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\tif (next_avail != rxq->next_avail) {\n+\t\trte_write32(rte_cpu_to_be_32(next_avail), rxq->qrx_tail);\n+\t\trxq->next_avail = next_avail;\n+\t}\n+}\n+\n+uint16_t\n+gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)\n+{\n+\tvolatile struct gve_rx_desc *rxr, *rxd;\n+\tstruct gve_rx_queue *rxq = rx_queue;\n+\tuint16_t rx_id = rxq->rx_tail;\n+\tstruct rte_mbuf *rxe;\n+\tuint16_t nb_rx, len;\n+\tuint64_t addr;\n+\tuint16_t i;\n+\n+\trxr = rxq->rx_desc_ring;\n+\tnb_rx = 0;\n+\n+\tfor (i = 0; i < nb_pkts; i++) {\n+\t\trxd = &rxr[rx_id];\n+\t\tif (GVE_SEQNO(rxd->flags_seq) != rxq->expected_seqno)\n+\t\t\tbreak;\n+\n+\t\tif (rxd->flags_seq & GVE_RXF_ERR)\n+\t\t\tcontinue;\n+\n+\t\tlen = rte_be_to_cpu_16(rxd->len) - GVE_RX_PAD;\n+\t\trxe = rxq->sw_ring[rx_id];\n+\t\tif (rxq->is_gqi_qpl) {\n+\t\t\taddr = (uint64_t)(rxq->qpl->mz->addr) + rx_id * PAGE_SIZE + GVE_RX_PAD;\n+\t\t\trte_memcpy((void *)((size_t)rxe->buf_addr + rxe->data_off),\n+\t\t\t\t   (void *)(size_t)addr, len);\n+\t\t}\n+\t\trxe->pkt_len = len;\n+\t\trxe->data_len = len;\n+\t\trxe->port = rxq->port_id;\n+\t\trxe->ol_flags = 0;\n+\n+\t\tif (rxd->flags_seq & GVE_RXF_TCP)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L4_TCP;\n+\t\tif (rxd->flags_seq & GVE_RXF_UDP)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L4_UDP;\n+\t\tif (rxd->flags_seq & GVE_RXF_IPV4)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L3_IPV4;\n+\t\tif (rxd->flags_seq & GVE_RXF_IPV6)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L3_IPV6;\n+\n+\t\tif (gve_needs_rss(rxd->flags_seq)) {\n+\t\t\trxe->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;\n+\t\t\trxe->hash.rss = rte_be_to_cpu_32(rxd->rss_hash);\n+\t\t}\n+\n+\t\trxq->expected_seqno = gve_next_seqno(rxq->expected_seqno);\n+\n+\t\trx_id++;\n+\t\tif (rx_id == rxq->nb_rx_desc)\n+\t\t\trx_id = 0;\n+\n+\t\trx_pkts[nb_rx] = rxe;\n+\t\tnb_rx++;\n+\t}\n+\n+\trxq->nb_avail += nb_rx;\n+\trxq->rx_tail = rx_id;\n+\n+\tif (rxq->nb_avail > rxq->free_thresh)\n+\t\tgve_rx_refill(rxq);\n+\n+\treturn nb_rx;\n+}\n+\n static inline void\n gve_reset_rxq(struct gve_rx_queue *rxq)\n {\ndiff --git a/drivers/net/gve/gve_tx.c b/drivers/net/gve/gve_tx.c\nindex b706b62e71..d94b1186a4 100644\n--- a/drivers/net/gve/gve_tx.c\n+++ b/drivers/net/gve/gve_tx.c\n@@ -5,6 +5,461 @@\n #include \"gve_ethdev.h\"\n #include \"base/gve_adminq.h\"\n \n+static inline void\n+gve_free_bulk_mbuf(struct rte_mbuf **txep, int num)\n+{\n+\tstruct rte_mbuf *m, *free[GVE_TX_MAX_FREE_SZ];\n+\tint nb_free = 0;\n+\tint i, s;\n+\n+\tif (unlikely(num == 0))\n+\t\treturn;\n+\n+\t/* Find the 1st mbuf which needs to be free */\n+\tfor (s = 0; s < num; s++) {\n+\t\tif (txep[s] != NULL) {\n+\t\t\tm = rte_pktmbuf_prefree_seg(txep[s]);\n+\t\t\tif (m != NULL)\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t}\n+\n+\tif (s == num)\n+\t\treturn;\n+\n+\tfree[0] = m;\n+\tnb_free = 1;\n+\tfor (i = s + 1; i < num; i++) {\n+\t\tif (likely(txep[i] != NULL)) {\n+\t\t\tm = rte_pktmbuf_prefree_seg(txep[i]);\n+\t\t\tif (likely(m != NULL)) {\n+\t\t\t\tif (likely(m->pool == free[0]->pool)) {\n+\t\t\t\t\tfree[nb_free++] = m;\n+\t\t\t\t} else {\n+\t\t\t\t\trte_mempool_put_bulk(free[0]->pool, (void *)free, nb_free);\n+\t\t\t\t\tfree[0] = m;\n+\t\t\t\t\tnb_free = 1;\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\ttxep[i] = NULL;\n+\t\t}\n+\t}\n+\trte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);\n+}\n+\n+static inline void\n+gve_tx_clean(struct gve_tx_queue *txq)\n+{\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint32_t start = txq->next_to_clean & mask;\n+\tuint32_t ntc, nb_clean, i;\n+\tstruct gve_tx_iovec *iov;\n+\n+\tntc = rte_be_to_cpu_32(rte_read32(txq->qtx_head));\n+\tntc = ntc & mask;\n+\n+\tif (ntc == start)\n+\t\treturn;\n+\n+\t/* if wrap around, free twice. */\n+\tif (ntc < start) {\n+\t\tnb_clean = txq->nb_tx_desc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tif (txq->is_gqi_qpl) {\n+\t\t\tfor (i = start; i < start + nb_clean; i++) {\n+\t\t\t\tiov = &txq->iov_ring[i];\n+\t\t\t\ttxq->fifo_avail += iov->iov_len;\n+\t\t\t\tiov->iov_base = 0;\n+\t\t\t\tiov->iov_len = 0;\n+\t\t\t}\n+\t\t} else {\n+\t\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\t}\n+\t\ttxq->nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\tif (start == txq->nb_tx_desc)\n+\t\t\tstart = 0;\n+\t\ttxq->next_to_clean += nb_clean;\n+\t}\n+\n+\tif (ntc > start) {\n+\t\tnb_clean = ntc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tif (txq->is_gqi_qpl) {\n+\t\t\tfor (i = start; i < start + nb_clean; i++) {\n+\t\t\t\tiov = &txq->iov_ring[i];\n+\t\t\t\ttxq->fifo_avail += iov->iov_len;\n+\t\t\t\tiov->iov_base = 0;\n+\t\t\t\tiov->iov_len = 0;\n+\t\t\t}\n+\t\t} else {\n+\t\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\t}\n+\t\ttxq->nb_free += nb_clean;\n+\t\ttxq->next_to_clean += nb_clean;\n+\t}\n+}\n+\n+static inline void\n+gve_tx_clean_swr_qpl(struct gve_tx_queue *txq)\n+{\n+\tuint32_t start = txq->sw_ntc;\n+\tuint32_t ntc, nb_clean;\n+\n+\tntc = txq->sw_tail;\n+\n+\tif (ntc == start)\n+\t\treturn;\n+\n+\t/* if wrap around, free twice. */\n+\tif (ntc < start) {\n+\t\tnb_clean = txq->nb_tx_desc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\n+\t\ttxq->sw_nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\tif (start == txq->nb_tx_desc)\n+\t\t\tstart = 0;\n+\t\ttxq->sw_ntc = start;\n+\t}\n+\n+\tif (ntc > start) {\n+\t\tnb_clean = ntc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\ttxq->sw_nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\ttxq->sw_ntc = start;\n+\t}\n+}\n+\n+static inline void\n+gve_tx_fill_pkt_desc(volatile union gve_tx_desc *desc, struct rte_mbuf *mbuf,\n+\t\t     uint8_t desc_cnt, uint16_t len, uint64_t addr)\n+{\n+\tuint64_t csum_l4 = mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK;\n+\tuint8_t l4_csum_offset = 0;\n+\tuint8_t l4_hdr_offset = 0;\n+\n+\tif (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\tcsum_l4 |= RTE_MBUF_F_TX_TCP_CKSUM;\n+\n+\tswitch (csum_l4) {\n+\tcase RTE_MBUF_F_TX_TCP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_tcp_hdr, cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\tcase RTE_MBUF_F_TX_UDP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_udp_hdr, dgram_cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\tcase RTE_MBUF_F_TX_SCTP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_sctp_hdr, cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\t}\n+\n+\tif (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\tdesc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;\n+\t\tdesc->pkt.l4_csum_offset = l4_csum_offset >> 1;\n+\t\tdesc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;\n+\t} else if (mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) {\n+\t\tdesc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;\n+\t\tdesc->pkt.l4_csum_offset = l4_csum_offset >> 1;\n+\t\tdesc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;\n+\t} else {\n+\t\tdesc->pkt.type_flags = GVE_TXD_STD;\n+\t\tdesc->pkt.l4_csum_offset = 0;\n+\t\tdesc->pkt.l4_hdr_offset = 0;\n+\t}\n+\tdesc->pkt.desc_cnt = desc_cnt;\n+\tdesc->pkt.len = rte_cpu_to_be_16(mbuf->pkt_len);\n+\tdesc->pkt.seg_len = rte_cpu_to_be_16(len);\n+\tdesc->pkt.seg_addr = rte_cpu_to_be_64(addr);\n+}\n+\n+static inline void\n+gve_tx_fill_seg_desc(volatile union gve_tx_desc *desc, uint64_t ol_flags,\n+\t\t      union gve_tx_offload tx_offload,\n+\t\t      uint16_t len, uint64_t addr)\n+{\n+\tdesc->seg.type_flags = GVE_TXD_SEG;\n+\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\tif (ol_flags & RTE_MBUF_F_TX_IPV6)\n+\t\t\tdesc->seg.type_flags |= GVE_TXSF_IPV6;\n+\t\tdesc->seg.l3_offset = tx_offload.l2_len >> 1;\n+\t\tdesc->seg.mss = rte_cpu_to_be_16(tx_offload.tso_segsz);\n+\t}\n+\tdesc->seg.seg_len = rte_cpu_to_be_16(len);\n+\tdesc->seg.seg_addr = rte_cpu_to_be_64(addr);\n+}\n+\n+static inline bool\n+is_fifo_avail(struct gve_tx_queue *txq, uint16_t len)\n+{\n+\tif (txq->fifo_avail < len)\n+\t\treturn false;\n+\t/* Don't split segment. */\n+\tif (txq->fifo_head + len > txq->fifo_size &&\n+\t    txq->fifo_size - txq->fifo_head + len > txq->fifo_avail)\n+\t\treturn false;\n+\treturn true;\n+}\n+static inline uint64_t\n+gve_tx_alloc_from_fifo(struct gve_tx_queue *txq, uint16_t tx_id, uint16_t len)\n+{\n+\tuint32_t head = txq->fifo_head;\n+\tuint32_t size = txq->fifo_size;\n+\tstruct gve_tx_iovec *iov;\n+\tuint32_t aligned_head;\n+\tuint32_t iov_len = 0;\n+\tuint64_t fifo_addr;\n+\n+\tiov = &txq->iov_ring[tx_id];\n+\n+\t/* Don't split segment */\n+\tif (head + len > size) {\n+\t\tiov_len += (size - head);\n+\t\thead = 0;\n+\t}\n+\n+\tfifo_addr = head;\n+\tiov_len += len;\n+\tiov->iov_base = head;\n+\n+\t/* Re-align to a cacheline for next head */\n+\thead += len;\n+\taligned_head = RTE_ALIGN(head, RTE_CACHE_LINE_SIZE);\n+\tiov_len += (aligned_head - head);\n+\tiov->iov_len = iov_len;\n+\n+\tif (aligned_head == txq->fifo_size)\n+\t\taligned_head = 0;\n+\ttxq->fifo_head = aligned_head;\n+\ttxq->fifo_avail -= iov_len;\n+\n+\treturn fifo_addr;\n+}\n+\n+static inline uint16_t\n+gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tunion gve_tx_offload tx_offload = {0};\n+\tvolatile union gve_tx_desc *txr, *txd;\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\tstruct rte_mbuf **sw_ring = txq->sw_ring;\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint16_t tx_id = txq->tx_tail & mask;\n+\tuint64_t ol_flags, addr, fifo_addr;\n+\tuint32_t tx_tail = txq->tx_tail;\n+\tstruct rte_mbuf *tx_pkt, *first;\n+\tuint16_t sw_id = txq->sw_tail;\n+\tuint16_t nb_used, i;\n+\tuint16_t nb_tx = 0;\n+\tuint32_t hlen;\n+\n+\ttxr = txq->tx_desc_ring;\n+\n+\tif (txq->nb_free < txq->free_thresh || txq->fifo_avail == 0)\n+\t\tgve_tx_clean(txq);\n+\n+\tif (txq->sw_nb_free < txq->free_thresh)\n+\t\tgve_tx_clean_swr_qpl(txq);\n+\n+\tfor (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {\n+\t\ttx_pkt = *tx_pkts++;\n+\t\tol_flags = tx_pkt->ol_flags;\n+\n+\t\tif (txq->sw_nb_free < tx_pkt->nb_segs) {\n+\t\t\tgve_tx_clean_swr_qpl(txq);\n+\t\t\tif (txq->sw_nb_free < tx_pkt->nb_segs)\n+\t\t\t\tgoto end_of_tx;\n+\t\t}\n+\n+\t\t/* Even for multi-segs, use 1 qpl buf for data */\n+\t\tnb_used = 1;\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\tnb_used++;\n+\n+\t\tif (txq->nb_free < nb_used)\n+\t\t\tgoto end_of_tx;\n+\n+\t\ttx_offload.l2_len = tx_pkt->l2_len;\n+\t\ttx_offload.l3_len = tx_pkt->l3_len;\n+\t\ttx_offload.l4_len = tx_pkt->l4_len;\n+\t\ttx_offload.tso_segsz = tx_pkt->tso_segsz;\n+\n+\t\tfirst = tx_pkt;\n+\t\ttxd = &txr[tx_id];\n+\t\thlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ?\n+\t\t\t(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :\n+\t\t\ttx_pkt->pkt_len;\n+\n+\t\tsw_ring[sw_id] = tx_pkt;\n+\t\tif (!is_fifo_avail(txq, hlen)) {\n+\t\t\tgve_tx_clean(txq);\n+\t\t\tif (!is_fifo_avail(txq, hlen))\n+\t\t\t\tgoto end_of_tx;\n+\t\t}\n+\t\taddr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off;\n+\t\tfifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);\n+\n+\t\t/* For TSO, check if there's enough fifo space for data first */\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\tif (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen)) {\n+\t\t\t\tgve_tx_clean(txq);\n+\t\t\t\tif (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen))\n+\t\t\t\t\tgoto end_of_tx;\n+\t\t\t}\n+\t\t}\n+\t\tif (tx_pkt->nb_segs == 1 || ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\trte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),\n+\t\t\t\t   (void *)(size_t)addr, hlen);\n+\t\telse\n+\t\t\trte_pktmbuf_read(tx_pkt, 0, hlen,\n+\t\t\t\t\t (void *)(size_t)(fifo_addr + txq->fifo_base));\n+\t\tgve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, fifo_addr);\n+\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\taddr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off + hlen;\n+\t\t\tfifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, tx_pkt->pkt_len - hlen);\n+\t\t\tif (tx_pkt->nb_segs == 1)\n+\t\t\t\trte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),\n+\t\t\t\t\t   (void *)(size_t)addr,\n+\t\t\t\t\t   tx_pkt->pkt_len - hlen);\n+\t\t\telse\n+\t\t\t\trte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,\n+\t\t\t\t\t\t (void *)(size_t)(fifo_addr + txq->fifo_base));\n+\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t     tx_pkt->pkt_len - hlen, fifo_addr);\n+\t\t}\n+\n+\t\t/* record mbuf in sw_ring for free */\n+\t\tfor (i = 1; i < first->nb_segs; i++) {\n+\t\t\tsw_id = (sw_id + 1) & mask;\n+\t\t\ttx_pkt = tx_pkt->next;\n+\t\t\tsw_ring[sw_id] = tx_pkt;\n+\t\t}\n+\n+\t\tsw_id = (sw_id + 1) & mask;\n+\t\ttx_id = (tx_id + 1) & mask;\n+\n+\t\ttxq->nb_free -= nb_used;\n+\t\ttxq->sw_nb_free -= first->nb_segs;\n+\t\ttx_tail += nb_used;\n+\t}\n+\n+end_of_tx:\n+\tif (nb_tx) {\n+\t\trte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);\n+\t\ttxq->tx_tail = tx_tail;\n+\t\ttxq->sw_tail = sw_id;\n+\t}\n+\n+\treturn nb_tx;\n+}\n+\n+static inline uint16_t\n+gve_tx_burst_ra(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tunion gve_tx_offload tx_offload = {0};\n+\tvolatile union gve_tx_desc *txr, *txd;\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\tstruct rte_mbuf **sw_ring = txq->sw_ring;\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint16_t tx_id = txq->tx_tail & mask;\n+\tuint32_t tx_tail = txq->tx_tail;\n+\tstruct rte_mbuf *tx_pkt, *first;\n+\tuint16_t nb_used, hlen, i;\n+\tuint64_t ol_flags, addr;\n+\tuint16_t nb_tx = 0;\n+\n+\ttxr = txq->tx_desc_ring;\n+\n+\tif (txq->nb_free < txq->free_thresh)\n+\t\tgve_tx_clean(txq);\n+\n+\tfor (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {\n+\t\ttx_pkt = *tx_pkts++;\n+\t\tol_flags = tx_pkt->ol_flags;\n+\n+\t\tnb_used = tx_pkt->nb_segs;\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\tnb_used++;\n+\n+\t\tif (txq->nb_free < nb_used)\n+\t\t\tgoto end_of_tx;\n+\n+\t\ttx_offload.l2_len = tx_pkt->l2_len;\n+\t\ttx_offload.l3_len = tx_pkt->l3_len;\n+\t\ttx_offload.l4_len = tx_pkt->l4_len;\n+\t\ttx_offload.tso_segsz = tx_pkt->tso_segsz;\n+\n+\t\tfirst = tx_pkt;\n+\t\ttxd = &txr[tx_id];\n+\n+\t\thlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ?\n+\t\t\t(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :\n+\t\t\ttx_pkt->pkt_len;\n+\t\t/*\n+\t\t * if tso, the driver needs to fill 2 descs for 1 mbuf\n+\t\t * so only put this mbuf into the 1st tx entry in sw ring\n+\t\t */\n+\t\tsw_ring[tx_id] = tx_pkt;\n+\t\taddr = rte_mbuf_data_iova(tx_pkt);\n+\t\tgve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, addr);\n+\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\taddr = rte_mbuf_data_iova(tx_pkt) + hlen;\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t     tx_pkt->data_len - hlen, addr);\n+\t\t}\n+\n+\t\tfor (i = 1; i < first->nb_segs; i++) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\ttx_pkt = tx_pkt->next;\n+\t\t\tsw_ring[tx_id] = tx_pkt;\n+\t\t\taddr = rte_mbuf_data_iova(tx_pkt);\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t     tx_pkt->data_len, addr);\n+\t\t}\n+\t\ttx_id = (tx_id + 1) & mask;\n+\n+\t\ttxq->nb_free -= nb_used;\n+\t\ttx_tail += nb_used;\n+\t}\n+\n+end_of_tx:\n+\tif (nb_tx) {\n+\t\trte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);\n+\t\ttxq->tx_tail = tx_tail;\n+\t}\n+\n+\treturn nb_tx;\n+}\n+\n+uint16_t\n+gve_tx_burst(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\n+\tif (txq->is_gqi_qpl)\n+\t\treturn gve_tx_burst_qpl(tx_queue, tx_pkts, nb_pkts);\n+\n+\treturn gve_tx_burst_ra(tx_queue, tx_pkts, nb_pkts);\n+}\n+\n static inline void\n gve_reset_txq(struct gve_tx_queue *txq)\n {\n",
    "prefixes": [
        "v6",
        "8/8"
    ]
}