Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/116731/?format=api
{ "id": 116731, "url": "http://patches.dpdk.org/api/patches/116731/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20220923093829.3019525-8-junfeng.guo@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20220923093829.3019525-8-junfeng.guo@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20220923093829.3019525-8-junfeng.guo@intel.com", "date": "2022-09-23T09:38:27", "name": "[v3,7/9] net/gve: add Rx/Tx support", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "d58d53a5ac0ab974d7fac4ee3298b3419309c4be", "submitter": { "id": 1785, "url": "http://patches.dpdk.org/api/people/1785/?format=api", "name": "Junfeng Guo", "email": "junfeng.guo@intel.com" }, "delegate": { "id": 319, "url": "http://patches.dpdk.org/api/users/319/?format=api", "username": "fyigit", "first_name": "Ferruh", "last_name": "Yigit", "email": "ferruh.yigit@amd.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20220923093829.3019525-8-junfeng.guo@intel.com/mbox/", "series": [ { "id": 24800, "url": "http://patches.dpdk.org/api/series/24800/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=24800", "date": "2022-09-23T09:38:20", "name": "introduce GVE PMD", "version": 3, "mbox": "http://patches.dpdk.org/series/24800/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/116731/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/116731/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 12A47A0544;\n\tFri, 23 Sep 2022 11:40:08 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 64B7F42BB4;\n\tFri, 23 Sep 2022 11:39:32 +0200 (CEST)", "from mga01.intel.com (mga01.intel.com [192.55.52.88])\n by mails.dpdk.org (Postfix) with ESMTP id 60C5E42BB4\n for <dev@dpdk.org>; Fri, 23 Sep 2022 11:39:30 +0200 (CEST)", "from orsmga008.jf.intel.com ([10.7.209.65])\n by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 23 Sep 2022 02:39:29 -0700", "from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104])\n by orsmga008.jf.intel.com with ESMTP; 23 Sep 2022 02:39:27 -0700" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1663925970; x=1695461970;\n h=from:to:cc:subject:date:message-id:in-reply-to:\n references:mime-version:content-transfer-encoding;\n bh=tMF7V4Hk/FZbRWZqI7yKLbNbrfzcyAsr37rozziy+xw=;\n b=JB3Ft3vqPpjWaPAXTY1Tza6nEQDWYIPSmVAtDMMd/8TTq5f1GEwlS0oo\n 5tYhqwHHfQxtnj4hMiFhot1n/W2HE/4c7SFwxHYTCj4aH6IudEzeCeDCc\n Y6jyK4vi58r1fW//V3ivHk1k9QyRIh95dnUTX9JeeDDzVHlUbTZdZsP+Q\n FhEvQ+gORUClJxXSsyQWD/voChePjKt7rqaTGi4nU9ptbMNQpBpYxvtmn\n uLdbfgkJinGsmp3awKuZDHtk6+VSJbVv59MbhWREJOj5EYMIjpGWzGdlE\n 0d61xk4giuVuRuVv1knZsBTWM7GM0/TK6gbKleWR0By71kppr/GQPiejT w==;", "X-IronPort-AV": [ "E=McAfee;i=\"6500,9779,10478\"; a=\"326885924\"", "E=Sophos;i=\"5.93,337,1654585200\"; d=\"scan'208\";a=\"326885924\"", "E=Sophos;i=\"5.93,337,1654585200\"; d=\"scan'208\";a=\"650891191\"" ], "X-ExtLoop1": "1", "From": "Junfeng Guo <junfeng.guo@intel.com>", "To": "qi.z.zhang@intel.com,\n\tjingjing.wu@intel.com", "Cc": "ferruh.yigit@xilinx.com, dev@dpdk.org, xiaoyun.li@intel.com,\n awogbemila@google.com, bruce.richardson@intel.com, xueqin.lin@intel.com,\n junfeng.guo@intel.com", "Subject": "[PATCH v3 7/9] net/gve: add Rx/Tx support", "Date": "Fri, 23 Sep 2022 17:38:27 +0800", "Message-Id": "<20220923093829.3019525-8-junfeng.guo@intel.com>", "X-Mailer": "git-send-email 2.34.1", "In-Reply-To": "<20220923093829.3019525-1-junfeng.guo@intel.com>", "References": "<20220829084127.934183-11-junfeng.guo@intel.com>\n <20220923093829.3019525-1-junfeng.guo@intel.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Add Rx/Tx of GQI_QPL queue format and GQI_RDA queue format.\n\nSigned-off-by: Xiaoyun Li <xiaoyun.li@intel.com>\nSigned-off-by: Junfeng Guo <junfeng.guo@intel.com>\n---\n doc/guides/nics/features/gve.ini | 2 +\n drivers/net/gve/gve_ethdev.c | 5 +\n drivers/net/gve/gve_ethdev.h | 16 ++\n drivers/net/gve/gve_rx.c | 143 ++++++++++\n drivers/net/gve/gve_tx.c | 455 +++++++++++++++++++++++++++++++\n 5 files changed, 621 insertions(+)", "diff": "diff --git a/doc/guides/nics/features/gve.ini b/doc/guides/nics/features/gve.ini\nindex fbff0a5462..38dc7024d6 100644\n--- a/doc/guides/nics/features/gve.ini\n+++ b/doc/guides/nics/features/gve.ini\n@@ -7,6 +7,8 @@\n Speed capabilities = Y\n Link status = Y\n MTU update = Y\n+TSO = Y\n+L4 checksum offload = Y\n Linux = Y\n x86-32 = Y\n x86-64 = Y\ndiff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c\nindex 72e7a78ace..dcf79ddb23 100644\n--- a/drivers/net/gve/gve_ethdev.c\n+++ b/drivers/net/gve/gve_ethdev.c\n@@ -583,6 +583,11 @@ gve_dev_init(struct rte_eth_dev *eth_dev)\n \tif (err)\n \t\treturn err;\n \n+\tif (gve_is_gqi(priv)) {\n+\t\teth_dev->rx_pkt_burst = gve_rx_burst;\n+\t\teth_dev->tx_pkt_burst = gve_tx_burst;\n+\t}\n+\n \teth_dev->data->mac_addrs = rte_zmalloc(\"gve_mac\", sizeof(struct rte_ether_addr), 0);\n \tif (!eth_dev->data->mac_addrs) {\n \t\tPMD_DRV_LOG(ERR, \"Failed to allocate memory to store mac address\");\ndiff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h\nindex 44e075e166..0624085517 100644\n--- a/drivers/net/gve/gve_ethdev.h\n+++ b/drivers/net/gve/gve_ethdev.h\n@@ -34,6 +34,18 @@ union gve_tx_desc {\n \tstruct gve_tx_seg_desc seg; /* subsequent descs for a packet */\n };\n \n+/* Offload features */\n+union gve_tx_offload {\n+\tuint64_t data;\n+\tstruct {\n+\t\tuint64_t l2_len:7; /* L2 (MAC) Header Length. */\n+\t\tuint64_t l3_len:9; /* L3 (IP) Header Length. */\n+\t\tuint64_t l4_len:8; /* L4 Header Length. */\n+\t\tuint64_t tso_segsz:16; /* TCP TSO segment size */\n+\t\t/* uint64_t unused : 24; */\n+\t};\n+};\n+\n struct gve_tx_iovec {\n \tuint32_t iov_base; /* offset in fifo */\n \tuint32_t iov_len;\n@@ -269,4 +281,8 @@ void gve_stop_tx_queues(struct rte_eth_dev *dev);\n \n void gve_stop_rx_queues(struct rte_eth_dev *dev);\n \n+uint16_t gve_rx_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);\n+\n+uint16_t gve_tx_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);\n+\n #endif /* _GVE_ETHDEV_H_ */\ndiff --git a/drivers/net/gve/gve_rx.c b/drivers/net/gve/gve_rx.c\nindex 90b3a52aca..e29f979a4e 100644\n--- a/drivers/net/gve/gve_rx.c\n+++ b/drivers/net/gve/gve_rx.c\n@@ -5,6 +5,149 @@\n #include \"gve_ethdev.h\"\n #include \"gve_adminq.h\"\n \n+static inline void\n+gve_rx_refill(struct gve_rx_queue *rxq)\n+{\n+\tuint16_t mask = rxq->nb_rx_desc - 1;\n+\tuint16_t idx = rxq->next_avail & mask;\n+\tuint32_t next_avail = rxq->next_avail;\n+\tuint16_t nb_alloc, i;\n+\tstruct rte_mbuf *nmb;\n+\tint diag;\n+\n+\t/* wrap around */\n+\tnb_alloc = rxq->nb_rx_desc - idx;\n+\tif (nb_alloc <= rxq->nb_avail) {\n+\t\tdiag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc);\n+\t\tif (diag < 0) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rte_pktmbuf_alloc(rxq->mpool);\n+\t\t\t\tif (!nmb)\n+\t\t\t\t\tbreak;\n+\t\t\t\trxq->sw_ring[idx + i] = nmb;\n+\t\t\t}\n+\t\t\tif (i != nb_alloc)\n+\t\t\t\tnb_alloc = i;\n+\t\t}\n+\t\trxq->nb_avail -= nb_alloc;\n+\t\tnext_avail += nb_alloc;\n+\n+\t\t/* queue page list mode doesn't need real refill. */\n+\t\tif (rxq->is_gqi_qpl) {\n+\t\t\tidx += nb_alloc;\n+\t\t} else {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rxq->sw_ring[idx];\n+\t\t\t\trxq->rx_data_ring[idx].addr =\n+\t\t\t\t\trte_cpu_to_be_64(rte_mbuf_data_iova(nmb));\n+\t\t\t\tidx++;\n+\t\t\t}\n+\t\t}\n+\t\tif (idx == rxq->nb_rx_desc)\n+\t\t\tidx = 0;\n+\t}\n+\n+\tif (rxq->nb_avail > 0) {\n+\t\tnb_alloc = rxq->nb_avail;\n+\t\tif (rxq->nb_rx_desc < idx + rxq->nb_avail)\n+\t\t\tnb_alloc = rxq->nb_rx_desc - idx;\n+\t\tdiag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc);\n+\t\tif (diag < 0) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rte_pktmbuf_alloc(rxq->mpool);\n+\t\t\t\tif (!nmb)\n+\t\t\t\t\tbreak;\n+\t\t\t\trxq->sw_ring[idx + i] = nmb;\n+\t\t\t}\n+\t\t\tnb_alloc = i;\n+\t\t}\n+\t\trxq->nb_avail -= nb_alloc;\n+\t\tnext_avail += nb_alloc;\n+\n+\t\tif (!rxq->is_gqi_qpl) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rxq->sw_ring[idx];\n+\t\t\t\trxq->rx_data_ring[idx].addr =\n+\t\t\t\t\trte_cpu_to_be_64(rte_mbuf_data_iova(nmb));\n+\t\t\t\tidx++;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\tif (next_avail != rxq->next_avail) {\n+\t\trte_write32(rte_cpu_to_be_32(next_avail), rxq->qrx_tail);\n+\t\trxq->next_avail = next_avail;\n+\t}\n+}\n+\n+uint16_t\n+gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)\n+{\n+\tvolatile struct gve_rx_desc *rxr, *rxd;\n+\tstruct gve_rx_queue *rxq = rx_queue;\n+\tuint16_t rx_id = rxq->rx_tail;\n+\tstruct rte_mbuf *rxe;\n+\tuint16_t nb_rx, len;\n+\tuint64_t addr;\n+\n+\trxr = rxq->rx_desc_ring;\n+\n+\tfor (nb_rx = 0; nb_rx < nb_pkts; nb_rx++) {\n+\t\trxd = &rxr[rx_id];\n+\t\tif (GVE_SEQNO(rxd->flags_seq) != rxq->expected_seqno)\n+\t\t\tbreak;\n+\n+\t\tif (rxd->flags_seq & GVE_RXF_ERR)\n+\t\t\tcontinue;\n+\n+\t\tlen = rte_be_to_cpu_16(rxd->len) - GVE_RX_PAD;\n+\t\trxe = rxq->sw_ring[rx_id];\n+\t\trxe->data_off = RTE_PKTMBUF_HEADROOM;\n+\t\tif (rxq->is_gqi_qpl) {\n+\t\t\taddr = (uint64_t)(rxq->qpl->mz->addr) + rx_id * PAGE_SIZE + GVE_RX_PAD;\n+\t\t\trte_memcpy((void *)((size_t)rxe->buf_addr + rxe->data_off),\n+\t\t\t\t (void *)(size_t)addr, len);\n+\t\t}\n+\t\trxe->nb_segs = 1;\n+\t\trxe->next = NULL;\n+\t\trxe->pkt_len = len;\n+\t\trxe->data_len = len;\n+\t\trxe->port = rxq->port_id;\n+\t\trxe->packet_type = 0;\n+\t\trxe->ol_flags = 0;\n+\n+\t\tif (rxd->flags_seq & GVE_RXF_TCP)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L4_TCP;\n+\t\tif (rxd->flags_seq & GVE_RXF_UDP)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L4_UDP;\n+\t\tif (rxd->flags_seq & GVE_RXF_IPV4)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L3_IPV4;\n+\t\tif (rxd->flags_seq & GVE_RXF_IPV6)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L3_IPV6;\n+\n+\t\tif (gve_needs_rss(rxd->flags_seq)) {\n+\t\t\trxe->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;\n+\t\t\trxe->hash.rss = rte_be_to_cpu_32(rxd->rss_hash);\n+\t\t}\n+\n+\t\trxq->expected_seqno = gve_next_seqno(rxq->expected_seqno);\n+\n+\t\trx_id++;\n+\t\tif (rx_id == rxq->nb_rx_desc)\n+\t\t\trx_id = 0;\n+\n+\t\trx_pkts[nb_rx] = rxe;\n+\t}\n+\n+\trxq->nb_avail += nb_rx;\n+\trxq->rx_tail = rx_id;\n+\n+\tif (rxq->nb_avail > rxq->free_thresh)\n+\t\tgve_rx_refill(rxq);\n+\n+\treturn nb_rx;\n+}\n+\n static inline void\n gve_reset_rxq(struct gve_rx_queue *rxq)\n {\ndiff --git a/drivers/net/gve/gve_tx.c b/drivers/net/gve/gve_tx.c\nindex b18e8123aa..6196c29e24 100644\n--- a/drivers/net/gve/gve_tx.c\n+++ b/drivers/net/gve/gve_tx.c\n@@ -5,6 +5,461 @@\n #include \"gve_ethdev.h\"\n #include \"gve_adminq.h\"\n \n+static inline void\n+gve_free_bulk_mbuf(struct rte_mbuf **txep, int num)\n+{\n+\tstruct rte_mbuf *m, *free[GVE_TX_MAX_FREE_SZ];\n+\tint nb_free = 0;\n+\tint i, s;\n+\n+\tif (unlikely(num == 0))\n+\t\treturn;\n+\n+\t/* Find the 1st mbuf which needs to be free */\n+\tfor (s = 0; s < num; s++) {\n+\t\tif (txep[s] != NULL) {\n+\t\t\tm = rte_pktmbuf_prefree_seg(txep[s]);\n+\t\t\tif (m != NULL)\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t}\n+\n+\tif (s == num)\n+\t\treturn;\n+\n+\tfree[0] = m;\n+\tnb_free = 1;\n+\tfor (i = s + 1; i < num; i++) {\n+\t\tif (likely(txep[i] != NULL)) {\n+\t\t\tm = rte_pktmbuf_prefree_seg(txep[i]);\n+\t\t\tif (likely(m != NULL)) {\n+\t\t\t\tif (likely(m->pool == free[0]->pool)) {\n+\t\t\t\t\tfree[nb_free++] = m;\n+\t\t\t\t} else {\n+\t\t\t\t\trte_mempool_put_bulk(free[0]->pool, (void *)free, nb_free);\n+\t\t\t\t\tfree[0] = m;\n+\t\t\t\t\tnb_free = 1;\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\ttxep[i] = NULL;\n+\t\t}\n+\t}\n+\trte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);\n+}\n+\n+static inline void\n+gve_tx_clean(struct gve_tx_queue *txq)\n+{\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint32_t start = txq->next_to_clean & mask;\n+\tuint32_t ntc, nb_clean, i;\n+\tstruct gve_tx_iovec *iov;\n+\n+\tntc = rte_be_to_cpu_32(rte_read32(txq->qtx_head));\n+\tntc = ntc & mask;\n+\n+\tif (ntc == start)\n+\t\treturn;\n+\n+\t/* if wrap around, free twice. */\n+\tif (ntc < start) {\n+\t\tnb_clean = txq->nb_tx_desc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tif (txq->is_gqi_qpl) {\n+\t\t\tfor (i = start; i < start + nb_clean; i++) {\n+\t\t\t\tiov = &txq->iov_ring[i];\n+\t\t\t\ttxq->fifo_avail += iov->iov_len;\n+\t\t\t\tiov->iov_base = 0;\n+\t\t\t\tiov->iov_len = 0;\n+\t\t\t}\n+\t\t} else {\n+\t\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\t}\n+\t\ttxq->nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\tif (start == txq->nb_tx_desc)\n+\t\t\tstart = 0;\n+\t\ttxq->next_to_clean += nb_clean;\n+\t}\n+\n+\tif (ntc > start) {\n+\t\tnb_clean = ntc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tif (txq->is_gqi_qpl) {\n+\t\t\tfor (i = start; i < start + nb_clean; i++) {\n+\t\t\t\tiov = &txq->iov_ring[i];\n+\t\t\t\ttxq->fifo_avail += iov->iov_len;\n+\t\t\t\tiov->iov_base = 0;\n+\t\t\t\tiov->iov_len = 0;\n+\t\t\t}\n+\t\t} else {\n+\t\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\t}\n+\t\ttxq->nb_free += nb_clean;\n+\t\ttxq->next_to_clean += nb_clean;\n+\t}\n+}\n+\n+static inline void\n+gve_tx_clean_swr_qpl(struct gve_tx_queue *txq)\n+{\n+\tuint32_t start = txq->sw_ntc;\n+\tuint32_t ntc, nb_clean;\n+\n+\tntc = txq->sw_tail;\n+\n+\tif (ntc == start)\n+\t\treturn;\n+\n+\t/* if wrap around, free twice. */\n+\tif (ntc < start) {\n+\t\tnb_clean = txq->nb_tx_desc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\n+\t\ttxq->sw_nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\tif (start == txq->nb_tx_desc)\n+\t\t\tstart = 0;\n+\t\ttxq->sw_ntc = start;\n+\t}\n+\n+\tif (ntc > start) {\n+\t\tnb_clean = ntc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\ttxq->sw_nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\ttxq->sw_ntc = start;\n+\t}\n+}\n+\n+static inline void\n+gve_tx_fill_pkt_desc(volatile union gve_tx_desc *desc, struct rte_mbuf *mbuf,\n+\t\t uint8_t desc_cnt, uint16_t len, uint64_t addr)\n+{\n+\tuint64_t csum_l4 = mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK;\n+\tuint8_t l4_csum_offset = 0;\n+\tuint8_t l4_hdr_offset = 0;\n+\n+\tif (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\tcsum_l4 |= RTE_MBUF_F_TX_TCP_CKSUM;\n+\n+\tswitch (csum_l4) {\n+\tcase RTE_MBUF_F_TX_TCP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_tcp_hdr, cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\tcase RTE_MBUF_F_TX_UDP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_udp_hdr, dgram_cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\tcase RTE_MBUF_F_TX_SCTP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_sctp_hdr, cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\t}\n+\n+\tif (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\tdesc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;\n+\t\tdesc->pkt.l4_csum_offset = l4_csum_offset >> 1;\n+\t\tdesc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;\n+\t} else if (mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) {\n+\t\tdesc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;\n+\t\tdesc->pkt.l4_csum_offset = l4_csum_offset >> 1;\n+\t\tdesc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;\n+\t} else {\n+\t\tdesc->pkt.type_flags = GVE_TXD_STD;\n+\t\tdesc->pkt.l4_csum_offset = 0;\n+\t\tdesc->pkt.l4_hdr_offset = 0;\n+\t}\n+\tdesc->pkt.desc_cnt = desc_cnt;\n+\tdesc->pkt.len = rte_cpu_to_be_16(mbuf->pkt_len);\n+\tdesc->pkt.seg_len = rte_cpu_to_be_16(len);\n+\tdesc->pkt.seg_addr = rte_cpu_to_be_64(addr);\n+}\n+\n+static inline void\n+gve_tx_fill_seg_desc(volatile union gve_tx_desc *desc, uint64_t ol_flags,\n+\t\t union gve_tx_offload tx_offload,\n+\t\t uint16_t len, uint64_t addr)\n+{\n+\tdesc->seg.type_flags = GVE_TXD_SEG;\n+\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\tif (ol_flags & RTE_MBUF_F_TX_IPV6)\n+\t\t\tdesc->seg.type_flags |= GVE_TXSF_IPV6;\n+\t\tdesc->seg.l3_offset = tx_offload.l2_len >> 1;\n+\t\tdesc->seg.mss = rte_cpu_to_be_16(tx_offload.tso_segsz);\n+\t}\n+\tdesc->seg.seg_len = rte_cpu_to_be_16(len);\n+\tdesc->seg.seg_addr = rte_cpu_to_be_64(addr);\n+}\n+\n+static inline bool\n+is_fifo_avail(struct gve_tx_queue *txq, uint16_t len)\n+{\n+\tif (txq->fifo_avail < len)\n+\t\treturn false;\n+\t/* Don't split segment. */\n+\tif (txq->fifo_head + len > txq->fifo_size &&\n+\t txq->fifo_size - txq->fifo_head + len > txq->fifo_avail)\n+\t\treturn false;\n+\treturn true;\n+}\n+static inline uint64_t\n+gve_tx_alloc_from_fifo(struct gve_tx_queue *txq, uint16_t tx_id, uint16_t len)\n+{\n+\tuint32_t head = txq->fifo_head;\n+\tuint32_t size = txq->fifo_size;\n+\tstruct gve_tx_iovec *iov;\n+\tuint32_t aligned_head;\n+\tuint32_t iov_len = 0;\n+\tuint64_t fifo_addr;\n+\n+\tiov = &txq->iov_ring[tx_id];\n+\n+\t/* Don't split segment */\n+\tif (head + len > size) {\n+\t\tiov_len += (size - head);\n+\t\thead = 0;\n+\t}\n+\n+\tfifo_addr = head;\n+\tiov_len += len;\n+\tiov->iov_base = head;\n+\n+\t/* Re-align to a cacheline for next head */\n+\thead += len;\n+\taligned_head = RTE_ALIGN(head, RTE_CACHE_LINE_SIZE);\n+\tiov_len += (aligned_head - head);\n+\tiov->iov_len = iov_len;\n+\n+\tif (aligned_head == txq->fifo_size)\n+\t\taligned_head = 0;\n+\ttxq->fifo_head = aligned_head;\n+\ttxq->fifo_avail -= iov_len;\n+\n+\treturn fifo_addr;\n+}\n+\n+static inline uint16_t\n+gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tunion gve_tx_offload tx_offload = {0};\n+\tvolatile union gve_tx_desc *txr, *txd;\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\tstruct rte_mbuf **sw_ring = txq->sw_ring;\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint16_t tx_id = txq->tx_tail & mask;\n+\tuint64_t ol_flags, addr, fifo_addr;\n+\tuint32_t tx_tail = txq->tx_tail;\n+\tstruct rte_mbuf *tx_pkt, *first;\n+\tuint16_t sw_id = txq->sw_tail;\n+\tuint16_t nb_used, i;\n+\tuint16_t nb_tx = 0;\n+\tuint32_t hlen;\n+\n+\ttxr = txq->tx_desc_ring;\n+\n+\tif (txq->nb_free < txq->free_thresh || txq->fifo_avail == 0)\n+\t\tgve_tx_clean(txq);\n+\n+\tif (txq->sw_nb_free < txq->free_thresh)\n+\t\tgve_tx_clean_swr_qpl(txq);\n+\n+\tfor (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {\n+\t\ttx_pkt = *tx_pkts++;\n+\t\tol_flags = tx_pkt->ol_flags;\n+\n+\t\tif (txq->sw_nb_free < tx_pkt->nb_segs) {\n+\t\t\tgve_tx_clean_swr_qpl(txq);\n+\t\t\tif (txq->sw_nb_free < tx_pkt->nb_segs)\n+\t\t\t\tgoto end_of_tx;\n+\t\t}\n+\n+\t\t/* Even for multi-segs, use 1 qpl buf for data */\n+\t\tnb_used = 1;\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\tnb_used++;\n+\n+\t\tif (txq->nb_free < nb_used)\n+\t\t\tgoto end_of_tx;\n+\n+\t\ttx_offload.l2_len = tx_pkt->l2_len;\n+\t\ttx_offload.l3_len = tx_pkt->l3_len;\n+\t\ttx_offload.l4_len = tx_pkt->l4_len;\n+\t\ttx_offload.tso_segsz = tx_pkt->tso_segsz;\n+\n+\t\tfirst = tx_pkt;\n+\t\ttxd = &txr[tx_id];\n+\t\thlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ?\n+\t\t\t(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :\n+\t\t\ttx_pkt->pkt_len;\n+\n+\t\tsw_ring[sw_id] = tx_pkt;\n+\t\tif (!is_fifo_avail(txq, hlen)) {\n+\t\t\tgve_tx_clean(txq);\n+\t\t\tif (!is_fifo_avail(txq, hlen))\n+\t\t\t\tgoto end_of_tx;\n+\t\t}\n+\t\taddr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off;\n+\t\tfifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);\n+\n+\t\t/* For TSO, check if there's enough fifo space for data first */\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\tif (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen)) {\n+\t\t\t\tgve_tx_clean(txq);\n+\t\t\t\tif (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen))\n+\t\t\t\t\tgoto end_of_tx;\n+\t\t\t}\n+\t\t}\n+\t\tif (tx_pkt->nb_segs == 1 || ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\trte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),\n+\t\t\t\t (void *)(size_t)addr, hlen);\n+\t\telse\n+\t\t\trte_pktmbuf_read(tx_pkt, 0, hlen,\n+\t\t\t\t\t (void *)(size_t)(fifo_addr + txq->fifo_base));\n+\t\tgve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, fifo_addr);\n+\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\taddr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off + hlen;\n+\t\t\tfifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, tx_pkt->pkt_len - hlen);\n+\t\t\tif (tx_pkt->nb_segs == 1)\n+\t\t\t\trte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),\n+\t\t\t\t\t (void *)(size_t)addr,\n+\t\t\t\t\t tx_pkt->pkt_len - hlen);\n+\t\t\telse\n+\t\t\t\trte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,\n+\t\t\t\t\t\t (void *)(size_t)(fifo_addr + txq->fifo_base));\n+\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t tx_pkt->pkt_len - hlen, fifo_addr);\n+\t\t}\n+\n+\t\t/* record mbuf in sw_ring for free */\n+\t\tfor (i = 1; i < first->nb_segs; i++) {\n+\t\t\tsw_id = (sw_id + 1) & mask;\n+\t\t\ttx_pkt = tx_pkt->next;\n+\t\t\tsw_ring[sw_id] = tx_pkt;\n+\t\t}\n+\n+\t\tsw_id = (sw_id + 1) & mask;\n+\t\ttx_id = (tx_id + 1) & mask;\n+\n+\t\ttxq->nb_free -= nb_used;\n+\t\ttxq->sw_nb_free -= first->nb_segs;\n+\t\ttx_tail += nb_used;\n+\t}\n+\n+end_of_tx:\n+\tif (nb_tx) {\n+\t\trte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);\n+\t\ttxq->tx_tail = tx_tail;\n+\t\ttxq->sw_tail = sw_id;\n+\t}\n+\n+\treturn nb_tx;\n+}\n+\n+static inline uint16_t\n+gve_tx_burst_ra(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tunion gve_tx_offload tx_offload = {0};\n+\tvolatile union gve_tx_desc *txr, *txd;\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\tstruct rte_mbuf **sw_ring = txq->sw_ring;\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint16_t tx_id = txq->tx_tail & mask;\n+\tuint32_t tx_tail = txq->tx_tail;\n+\tstruct rte_mbuf *tx_pkt, *first;\n+\tuint16_t nb_used, hlen, i;\n+\tuint64_t ol_flags, addr;\n+\tuint16_t nb_tx = 0;\n+\n+\ttxr = txq->tx_desc_ring;\n+\n+\tif (txq->nb_free < txq->free_thresh)\n+\t\tgve_tx_clean(txq);\n+\n+\tfor (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {\n+\t\ttx_pkt = *tx_pkts++;\n+\t\tol_flags = tx_pkt->ol_flags;\n+\n+\t\tnb_used = tx_pkt->nb_segs;\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\tnb_used++;\n+\n+\t\tif (txq->nb_free < nb_used)\n+\t\t\tgoto end_of_tx;\n+\n+\t\ttx_offload.l2_len = tx_pkt->l2_len;\n+\t\ttx_offload.l3_len = tx_pkt->l3_len;\n+\t\ttx_offload.l4_len = tx_pkt->l4_len;\n+\t\ttx_offload.tso_segsz = tx_pkt->tso_segsz;\n+\n+\t\tfirst = tx_pkt;\n+\t\ttxd = &txr[tx_id];\n+\n+\t\thlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ?\n+\t\t\t(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :\n+\t\t\ttx_pkt->pkt_len;\n+\t\t/*\n+\t\t * if tso, the driver needs to fill 2 descs for 1 mbuf\n+\t\t * so only put this mbuf into the 1st tx entry in sw ring\n+\t\t */\n+\t\tsw_ring[tx_id] = tx_pkt;\n+\t\taddr = rte_mbuf_data_iova(tx_pkt);\n+\t\tgve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, addr);\n+\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\taddr = rte_mbuf_data_iova(tx_pkt) + hlen;\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t tx_pkt->data_len - hlen, addr);\n+\t\t}\n+\n+\t\tfor (i = 1; i < first->nb_segs; i++) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\ttx_pkt = tx_pkt->next;\n+\t\t\tsw_ring[tx_id] = tx_pkt;\n+\t\t\taddr = rte_mbuf_data_iova(tx_pkt);\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t tx_pkt->data_len, addr);\n+\t\t}\n+\t\ttx_id = (tx_id + 1) & mask;\n+\n+\t\ttxq->nb_free -= nb_used;\n+\t\ttx_tail += nb_used;\n+\t}\n+\n+end_of_tx:\n+\tif (nb_tx) {\n+\t\trte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);\n+\t\ttxq->tx_tail = tx_tail;\n+\t}\n+\n+\treturn nb_tx;\n+}\n+\n+uint16_t\n+gve_tx_burst(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\n+\tif (txq->is_gqi_qpl)\n+\t\treturn gve_tx_burst_qpl(tx_queue, tx_pkts, nb_pkts);\n+\n+\treturn gve_tx_burst_ra(tx_queue, tx_pkts, nb_pkts);\n+}\n+\n static inline void\n gve_reset_txq(struct gve_tx_queue *txq)\n {\n", "prefixes": [ "v3", "7/9" ] }