Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/114457/?format=api
{ "id": 114457, "url": "http://patches.dpdk.org/api/patches/114457/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20220729193042.2764633-8-xiaoyun.li@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20220729193042.2764633-8-xiaoyun.li@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20220729193042.2764633-8-xiaoyun.li@intel.com", "date": "2022-07-29T19:30:39", "name": "[07/10] net/gve: add Rx/Tx support", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "a03a2f24d42eab35f6dd7d4048fd001943bcbe5b", "submitter": { "id": 798, "url": "http://patches.dpdk.org/api/people/798/?format=api", "name": "Li, Xiaoyun", "email": "xiaoyun.li@intel.com" }, "delegate": { "id": 319, "url": "http://patches.dpdk.org/api/users/319/?format=api", "username": "fyigit", "first_name": "Ferruh", "last_name": "Yigit", "email": "ferruh.yigit@amd.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20220729193042.2764633-8-xiaoyun.li@intel.com/mbox/", "series": [ { "id": 24137, "url": "http://patches.dpdk.org/api/series/24137/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=24137", "date": "2022-07-29T19:30:32", "name": "introduce GVE PMD", "version": 1, "mbox": "http://patches.dpdk.org/series/24137/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/114457/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/114457/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id B0A0FA00C4;\n\tFri, 29 Jul 2022 21:31:52 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id CEC0342C75;\n\tFri, 29 Jul 2022 21:31:17 +0200 (CEST)", "from mga12.intel.com (mga12.intel.com [192.55.52.136])\n by mails.dpdk.org (Postfix) with ESMTP id 641D842C71\n for <dev@dpdk.org>; Fri, 29 Jul 2022 21:31:15 +0200 (CEST)", "from orsmga006.jf.intel.com ([10.7.209.51])\n by fmsmga106.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 29 Jul 2022 12:31:14 -0700", "from silpixa00399779.ir.intel.com (HELO\n silpixa00399779.ger.corp.intel.com) ([10.237.223.111])\n by orsmga006.jf.intel.com with ESMTP; 29 Jul 2022 12:31:13 -0700" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1659123075; x=1690659075;\n h=from:to:cc:subject:date:message-id:in-reply-to:\n references:mime-version:content-transfer-encoding;\n bh=h1cDiXzdjfMwxgtfIHm8yIv8qZvmkGKCFe8bzDo/A0k=;\n b=mfSIlIZjy13mawxiK9OeGApXx6u/SqyI/96K4QoSzPxaVAftFdYG0ckY\n EwguNdvmd27hVXzwzauPNwHFHdHrVKydIXZs1Q4w6/SmrLZxGP6PC25qf\n Tp30F8tXjidVJxhUWtaerMdbtYFfbPJEhUoByF/KjyW53SN6yqf0AOsjp\n IL0KOb9T/v39ry5h5I9xWPxZ40KP7g0kSOZFw0Rg41D19UmL3DgStLyrU\n 2bYLUZLsYTMAvvwY9pHdnKBhWHBNsKF+qU5quJNnq2+KJP1VbDKDTLMLw\n +kT3pii//TFqVbriDKZnGNbg7TR7444dQSP511jg5anC3WvKHheP7Ogyv g==;", "X-IronPort-AV": [ "E=McAfee;i=\"6400,9594,10423\"; a=\"268602960\"", "E=Sophos;i=\"5.93,201,1654585200\"; d=\"scan'208\";a=\"268602960\"", "E=Sophos;i=\"5.93,201,1654585200\"; d=\"scan'208\";a=\"577059571\"" ], "X-ExtLoop1": "1", "From": "Xiaoyun Li <xiaoyun.li@intel.com>", "To": "junfeng.guo@intel.com, qi.z.zhang@intel.com, awogbemila@google.com,\n bruce.richardson@intel.com", "Cc": "dev@dpdk.org,\n\tXiaoyun Li <xiaoyun.li@intel.com>", "Subject": "[PATCH 07/10] net/gve: add Rx/Tx support", "Date": "Fri, 29 Jul 2022 19:30:39 +0000", "Message-Id": "<20220729193042.2764633-8-xiaoyun.li@intel.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20220729193042.2764633-1-xiaoyun.li@intel.com>", "References": "<20220729193042.2764633-1-xiaoyun.li@intel.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Add Rx/Tx of GQI_QPL queue format and GQI_RDA queue format.\n\nSigned-off-by: Xiaoyun Li <xiaoyun.li@intel.com>\n---\n drivers/net/gve/gve.h | 17 ++\n drivers/net/gve/gve_ethdev.c | 5 +\n drivers/net/gve/gve_rx.c | 143 +++++++++++\n drivers/net/gve/gve_tx.c | 452 +++++++++++++++++++++++++++++++++++\n 4 files changed, 617 insertions(+)", "diff": "diff --git a/drivers/net/gve/gve.h b/drivers/net/gve/gve.h\nindex a53a852a5f..7f4d0e37f3 100644\n--- a/drivers/net/gve/gve.h\n+++ b/drivers/net/gve/gve.h\n@@ -25,6 +25,7 @@\n \n #define GVE_DEFAULT_RX_FREE_THRESH 512\n #define GVE_DEFAULT_TX_FREE_THRESH 256\n+#define GVE_TX_MAX_FREE_SZ 512\n \n /* PTYPEs are always 10 bits. */\n #define GVE_NUM_PTYPES\t1024\n@@ -45,6 +46,18 @@ union gve_tx_desc {\n \tstruct gve_tx_seg_desc seg; /* subsequent descs for a packet */\n };\n \n+/* Offload features */\n+union gve_tx_offload {\n+\tuint64_t data;\n+\tstruct {\n+\t\tuint64_t l2_len:7; /* L2 (MAC) Header Length. */\n+\t\tuint64_t l3_len:9; /* L3 (IP) Header Length. */\n+\t\tuint64_t l4_len:8; /* L4 Header Length. */\n+\t\tuint64_t tso_segsz:16; /* TCP TSO segment size */\n+\t\t/* uint64_t unused : 24; */\n+\t};\n+};\n+\n struct gve_tx_iovec {\n \tuint32_t iov_base; /* offset in fifo */\n \tuint32_t iov_len;\n@@ -298,4 +311,8 @@ void gve_stop_tx_queues(struct rte_eth_dev *dev);\n \n void gve_stop_rx_queues(struct rte_eth_dev *dev);\n \n+uint16_t gve_rx_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);\n+\n+uint16_t gve_tx_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);\n+\n #endif /* _GVE_H_ */\ndiff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c\nindex 5201398664..5ebe2c30ea 100644\n--- a/drivers/net/gve/gve_ethdev.c\n+++ b/drivers/net/gve/gve_ethdev.c\n@@ -583,6 +583,11 @@ gve_dev_init(struct rte_eth_dev *eth_dev)\n \tif (err)\n \t\treturn err;\n \n+\tif (gve_is_gqi(priv)) {\n+\t\teth_dev->rx_pkt_burst = gve_rx_burst;\n+\t\teth_dev->tx_pkt_burst = gve_tx_burst;\n+\t}\n+\n \teth_dev->data->mac_addrs = rte_zmalloc(\"gve_mac\", sizeof(struct rte_ether_addr), 0);\n \tif (!eth_dev->data->mac_addrs) {\n \t\tPMD_INIT_LOG(ERR, \"Failed to allocate memory to store mac address\");\ndiff --git a/drivers/net/gve/gve_rx.c b/drivers/net/gve/gve_rx.c\nindex 7298b4cc86..8f560ae592 100644\n--- a/drivers/net/gve/gve_rx.c\n+++ b/drivers/net/gve/gve_rx.c\n@@ -5,6 +5,149 @@\n #include \"gve.h\"\n #include \"gve_adminq.h\"\n \n+static inline void\n+gve_rx_refill(struct gve_rx_queue *rxq)\n+{\n+\tuint16_t mask = rxq->nb_rx_desc - 1;\n+\tuint16_t idx = rxq->next_avail & mask;\n+\tuint32_t next_avail = rxq->next_avail;\n+\tuint16_t nb_alloc, i;\n+\tstruct rte_mbuf *nmb;\n+\tint diag;\n+\n+\t/* wrap around */\n+\tnb_alloc = rxq->nb_rx_desc - idx;\n+\tif (nb_alloc <= rxq->nb_avail) {\n+\t\tdiag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc);\n+\t\tif (diag < 0) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rte_pktmbuf_alloc(rxq->mpool);\n+\t\t\t\tif (!nmb)\n+\t\t\t\t\tbreak;\n+\t\t\t\trxq->sw_ring[idx + i] = nmb;\n+\t\t\t}\n+\t\t\tif (i != nb_alloc)\n+\t\t\t\tnb_alloc = i;\n+\t\t}\n+\t\trxq->nb_avail -= nb_alloc;\n+\t\tnext_avail += nb_alloc;\n+\n+\t\t/* queue page list mode doesn't need real refill. */\n+\t\tif (rxq->is_gqi_qpl) {\n+\t\t\tidx += nb_alloc;\n+\t\t} else {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rxq->sw_ring[idx];\n+\t\t\t\trxq->rx_data_ring[idx].addr =\n+\t\t\t\t\trte_cpu_to_be_64(rte_mbuf_data_iova(nmb));\n+\t\t\t\tidx++;\n+\t\t\t}\n+\t\t}\n+\t\tif (idx == rxq->nb_rx_desc)\n+\t\t\tidx = 0;\n+\t}\n+\n+\tif (rxq->nb_avail > 0) {\n+\t\tnb_alloc = rxq->nb_avail;\n+\t\tif (rxq->nb_rx_desc < idx + rxq->nb_avail)\n+\t\t\tnb_alloc = rxq->nb_rx_desc - idx;\n+\t\tdiag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc);\n+\t\tif (diag < 0) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rte_pktmbuf_alloc(rxq->mpool);\n+\t\t\t\tif (!nmb)\n+\t\t\t\t\tbreak;\n+\t\t\t\trxq->sw_ring[idx + i] = nmb;\n+\t\t\t}\n+\t\t\tnb_alloc = i;\n+\t\t}\n+\t\trxq->nb_avail -= nb_alloc;\n+\t\tnext_avail += nb_alloc;\n+\n+\t\tif (!rxq->is_gqi_qpl) {\n+\t\t\tfor (i = 0; i < nb_alloc; i++) {\n+\t\t\t\tnmb = rxq->sw_ring[idx];\n+\t\t\t\trxq->rx_data_ring[idx].addr =\n+\t\t\t\t\trte_cpu_to_be_64(rte_mbuf_data_iova(nmb));\n+\t\t\t\tidx++;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\tif (next_avail != rxq->next_avail) {\n+\t\trte_write32(rte_cpu_to_be_32(next_avail), rxq->qrx_tail);\n+\t\trxq->next_avail = next_avail;\n+\t}\n+}\n+\n+uint16_t\n+gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)\n+{\n+\tvolatile struct gve_rx_desc *rxr, *rxd;\n+\tstruct gve_rx_queue *rxq = rx_queue;\n+\tuint16_t rx_id = rxq->rx_tail;\n+\tstruct rte_mbuf *rxe;\n+\tuint16_t nb_rx, len;\n+\tuint64_t addr;\n+\n+\trxr = rxq->rx_desc_ring;\n+\n+\tfor (nb_rx = 0; nb_rx < nb_pkts; nb_rx++) {\n+\t\trxd = &rxr[rx_id];\n+\t\tif (GVE_SEQNO(rxd->flags_seq) != rxq->expected_seqno)\n+\t\t\tbreak;\n+\n+\t\tif (rxd->flags_seq & GVE_RXF_ERR)\n+\t\t\tcontinue;\n+\n+\t\tlen = rte_be_to_cpu_16(rxd->len) - GVE_RX_PAD;\n+\t\trxe = rxq->sw_ring[rx_id];\n+\t\trxe->data_off = RTE_PKTMBUF_HEADROOM;\n+\t\tif (rxq->is_gqi_qpl) {\n+\t\t\taddr = (uint64_t)(rxq->qpl->mz->addr) + rx_id * PAGE_SIZE + GVE_RX_PAD;\n+\t\t\trte_memcpy((void *)((uint64_t)rxe->buf_addr + rxe->data_off),\n+\t\t\t\t (void *)addr, len);\n+\t\t}\n+\t\trxe->nb_segs = 1;\n+\t\trxe->next = NULL;\n+\t\trxe->pkt_len = len;\n+\t\trxe->data_len = len;\n+\t\trxe->port = rxq->port_id;\n+\t\trxe->packet_type = 0;\n+\t\trxe->ol_flags = 0;\n+\n+\t\tif (rxd->flags_seq & GVE_RXF_TCP)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L4_TCP;\n+\t\tif (rxd->flags_seq & GVE_RXF_UDP)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L4_UDP;\n+\t\tif (rxd->flags_seq & GVE_RXF_IPV4)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L3_IPV4;\n+\t\tif (rxd->flags_seq & GVE_RXF_IPV6)\n+\t\t\trxe->packet_type |= RTE_PTYPE_L3_IPV6;\n+\n+\t\tif (gve_needs_rss(rxd->flags_seq)) {\n+\t\t\trxe->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;\n+\t\t\trxe->hash.rss = rte_be_to_cpu_32(rxd->rss_hash);\n+\t\t}\n+\n+\t\trxq->expected_seqno = gve_next_seqno(rxq->expected_seqno);\n+\n+\t\trx_id++;\n+\t\tif (rx_id == rxq->nb_rx_desc)\n+\t\t\trx_id = 0;\n+\n+\t\trx_pkts[nb_rx] = rxe;\n+\t}\n+\n+\trxq->nb_avail += nb_rx;\n+\trxq->rx_tail = rx_id;\n+\n+\tif (rxq->nb_avail > rxq->free_thresh)\n+\t\tgve_rx_refill(rxq);\n+\n+\treturn nb_rx;\n+}\n+\n static inline void\n gve_reset_rxq(struct gve_rx_queue *rxq)\n {\ndiff --git a/drivers/net/gve/gve_tx.c b/drivers/net/gve/gve_tx.c\nindex 947c9d1627..2dc3411672 100644\n--- a/drivers/net/gve/gve_tx.c\n+++ b/drivers/net/gve/gve_tx.c\n@@ -5,6 +5,458 @@\n #include \"gve.h\"\n #include \"gve_adminq.h\"\n \n+static inline void\n+gve_free_bulk_mbuf(struct rte_mbuf **txep, int num)\n+{\n+\tstruct rte_mbuf *m, *free[GVE_TX_MAX_FREE_SZ];\n+\tint nb_free = 0;\n+\tint i, s;\n+\n+\tif (unlikely(num == 0))\n+\t\treturn;\n+\n+\t/* Find the 1st mbuf which needs to be free */\n+\tfor (s = 0; s < num; s++) {\n+\t\tif (txep[s] != NULL) {\n+\t\t\tm = rte_pktmbuf_prefree_seg(txep[s]);\n+\t\t\tif (m != NULL)\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t}\n+\n+\tif (s == num)\n+\t\treturn;\n+\n+\tfree[0] = m;\n+\tnb_free = 1;\n+\tfor (i = s + 1; i < num; i++) {\n+\t\tif (likely(txep[i] != NULL)) {\n+\t\t\tm = rte_pktmbuf_prefree_seg(txep[i]);\n+\t\t\tif (likely(m != NULL)) {\n+\t\t\t\tif (likely(m->pool == free[0]->pool)) {\n+\t\t\t\t\tfree[nb_free++] = m;\n+\t\t\t\t} else {\n+\t\t\t\t\trte_mempool_put_bulk(free[0]->pool, (void *)free, nb_free);\n+\t\t\t\t\tfree[0] = m;\n+\t\t\t\t\tnb_free = 1;\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\ttxep[i] = NULL;\n+\t\t}\n+\t}\n+\trte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);\n+}\n+\n+static inline void\n+gve_tx_clean(struct gve_tx_queue *txq)\n+{\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint32_t start = txq->next_to_clean & mask;\n+\tuint32_t ntc, nb_clean, i;\n+\tstruct gve_tx_iovec *iov;\n+\n+\tntc = rte_be_to_cpu_32(rte_read32(txq->qtx_head));\n+\tntc = ntc & mask;\n+\n+\tif (ntc == start)\n+\t\treturn;\n+\n+\t/* if wrap around, free twice. */\n+\tif (ntc < start) {\n+\t\tnb_clean = txq->nb_tx_desc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tif (txq->is_gqi_qpl) {\n+\t\t\tfor (i = start; i < start + nb_clean; i++) {\n+\t\t\t\tiov = &txq->iov_ring[i];\n+\t\t\t\ttxq->fifo_avail += iov->iov_len;\n+\t\t\t\tiov->iov_base = 0;\n+\t\t\t\tiov->iov_len = 0;\n+\t\t\t}\n+\t\t} else {\n+\t\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\t}\n+\t\ttxq->nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\tif (start == txq->nb_tx_desc)\n+\t\t\tstart = 0;\n+\t\ttxq->next_to_clean += nb_clean;\n+\t}\n+\n+\tif (ntc > start) {\n+\t\tnb_clean = ntc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tif (txq->is_gqi_qpl) {\n+\t\t\tfor (i = start; i < start + nb_clean; i++) {\n+\t\t\t\tiov = &txq->iov_ring[i];\n+\t\t\t\ttxq->fifo_avail += iov->iov_len;\n+\t\t\t\tiov->iov_base = 0;\n+\t\t\t\tiov->iov_len = 0;\n+\t\t\t}\n+\t\t} else {\n+\t\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\t}\n+\t\ttxq->nb_free += nb_clean;\n+\t\ttxq->next_to_clean += nb_clean;\n+\t}\n+}\n+\n+static inline void\n+gve_tx_clean_swr_qpl(struct gve_tx_queue *txq)\n+{\n+\tuint32_t start = txq->sw_ntc;\n+\tuint32_t ntc, nb_clean;\n+\n+\tntc = txq->sw_tail;\n+\n+\tif (ntc == start)\n+\t\treturn;\n+\n+\t/* if wrap around, free twice. */\n+\tif (ntc < start) {\n+\t\tnb_clean = txq->nb_tx_desc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\n+\t\ttxq->sw_nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\tif (start == txq->nb_tx_desc)\n+\t\t\tstart = 0;\n+\t\ttxq->sw_ntc = start;\n+\t}\n+\n+\tif (ntc > start) {\n+\t\tnb_clean = ntc - start;\n+\t\tif (nb_clean > GVE_TX_MAX_FREE_SZ)\n+\t\t\tnb_clean = GVE_TX_MAX_FREE_SZ;\n+\t\tgve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);\n+\t\ttxq->sw_nb_free += nb_clean;\n+\t\tstart += nb_clean;\n+\t\ttxq->sw_ntc = start;\n+\t}\n+}\n+\n+static inline void\n+gve_tx_fill_pkt_desc(volatile union gve_tx_desc *desc, struct rte_mbuf *mbuf,\n+\t\t uint8_t desc_cnt, uint16_t len, uint64_t addr)\n+{\n+\tuint64_t csum_l4 = mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK;\n+\tuint8_t l4_csum_offset = 0;\n+\tuint8_t l4_hdr_offset = 0;\n+\n+\tif (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\tcsum_l4 |= RTE_MBUF_F_TX_TCP_CKSUM;\n+\n+\tswitch (csum_l4) {\n+\tcase RTE_MBUF_F_TX_TCP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_tcp_hdr, cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\tcase RTE_MBUF_F_TX_UDP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_udp_hdr, dgram_cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\tcase RTE_MBUF_F_TX_SCTP_CKSUM:\n+\t\tl4_csum_offset = offsetof(struct rte_sctp_hdr, cksum);\n+\t\tl4_hdr_offset = mbuf->l2_len + mbuf->l3_len;\n+\t\tbreak;\n+\t}\n+\n+\tif (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\tdesc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;\n+\t\tdesc->pkt.l4_csum_offset = l4_csum_offset >> 1;\n+\t\tdesc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;\n+\t} else if (mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) {\n+\t\tdesc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;\n+\t\tdesc->pkt.l4_csum_offset = l4_csum_offset >> 1;\n+\t\tdesc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;\n+\t} else {\n+\t\tdesc->pkt.type_flags = GVE_TXD_STD;\n+\t\tdesc->pkt.l4_csum_offset = 0;\n+\t\tdesc->pkt.l4_hdr_offset = 0;\n+\t}\n+\tdesc->pkt.desc_cnt = desc_cnt;\n+\tdesc->pkt.len = rte_cpu_to_be_16(mbuf->pkt_len);\n+\tdesc->pkt.seg_len = rte_cpu_to_be_16(len);\n+\tdesc->pkt.seg_addr = rte_cpu_to_be_64(addr);\n+}\n+\n+static inline void\n+gve_tx_fill_seg_desc(volatile union gve_tx_desc *desc, uint64_t ol_flags,\n+\t\t union gve_tx_offload tx_offload,\n+\t\t uint16_t len, uint64_t addr)\n+{\n+\tdesc->seg.type_flags = GVE_TXD_SEG;\n+\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\tif (ol_flags & RTE_MBUF_F_TX_IPV6)\n+\t\t\tdesc->seg.type_flags |= GVE_TXSF_IPV6;\n+\t\tdesc->seg.l3_offset = tx_offload.l2_len >> 1;\n+\t\tdesc->seg.mss = rte_cpu_to_be_16(tx_offload.tso_segsz);\n+\t}\n+\tdesc->seg.seg_len = rte_cpu_to_be_16(len);\n+\tdesc->seg.seg_addr = rte_cpu_to_be_64(addr);\n+}\n+\n+static inline bool\n+is_fifo_avail(struct gve_tx_queue *txq, uint16_t len)\n+{\n+\tif (txq->fifo_avail < len)\n+\t\treturn false;\n+\t/* Don't split segment. */\n+\tif (txq->fifo_head + len > txq->fifo_size &&\n+\t txq->fifo_size - txq->fifo_head + len > txq->fifo_avail)\n+\t\treturn false;\n+\treturn true;\n+}\n+static inline uint64_t\n+gve_tx_alloc_from_fifo(struct gve_tx_queue *txq, uint16_t tx_id, uint16_t len)\n+{\n+\tuint32_t head = txq->fifo_head;\n+\tuint32_t size = txq->fifo_size;\n+\tstruct gve_tx_iovec *iov;\n+\tuint32_t aligned_head;\n+\tuint32_t iov_len = 0;\n+\tuint64_t fifo_addr;\n+\n+\tiov = &txq->iov_ring[tx_id];\n+\n+\t/* Don't split segment */\n+\tif (head + len > size) {\n+\t\tiov_len += (size - head);\n+\t\thead = 0;\n+\t}\n+\n+\tfifo_addr = head;\n+\tiov_len += len;\n+\tiov->iov_base = head;\n+\n+\t/* Re-align to a cacheline for next head */\n+\thead += len;\n+\taligned_head = RTE_ALIGN(head, RTE_CACHE_LINE_SIZE);\n+\tiov_len += (aligned_head - head);\n+\tiov->iov_len = iov_len;\n+\n+\tif (aligned_head == txq->fifo_size)\n+\t\taligned_head = 0;\n+\ttxq->fifo_head = aligned_head;\n+\ttxq->fifo_avail -= iov_len;\n+\n+\treturn fifo_addr;\n+}\n+\n+static inline uint16_t\n+gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tunion gve_tx_offload tx_offload = {0};\n+\tvolatile union gve_tx_desc *txr, *txd;\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\tstruct rte_mbuf **sw_ring = txq->sw_ring;\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint16_t tx_id = txq->tx_tail & mask;\n+\tuint64_t ol_flags, addr, fifo_addr;\n+\tuint32_t tx_tail = txq->tx_tail;\n+\tstruct rte_mbuf *tx_pkt, *first;\n+\tuint16_t sw_id = txq->sw_tail;\n+\tuint16_t nb_used, i;\n+\tuint16_t nb_tx = 0;\n+\tuint32_t hlen;\n+\n+\ttxr = txq->tx_desc_ring;\n+\n+\tif (txq->nb_free < txq->free_thresh || txq->fifo_avail == 0)\n+\t\tgve_tx_clean(txq);\n+\n+\tif (txq->sw_nb_free < txq->free_thresh)\n+\t\tgve_tx_clean_swr_qpl(txq);\n+\n+\tfor (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {\n+\t\ttx_pkt = *tx_pkts++;\n+\t\tol_flags = tx_pkt->ol_flags;\n+\n+\t\tif (txq->sw_nb_free < tx_pkt->nb_segs) {\n+\t\t\tgve_tx_clean_swr_qpl(txq);\n+\t\t\tif (txq->sw_nb_free < tx_pkt->nb_segs)\n+\t\t\t\tgoto end_of_tx;\n+\t\t}\n+\n+\t\t/* Even for multi-segs, use 1 qpl buf for data */\n+\t\tnb_used = 1;\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\tnb_used++;\n+\n+\t\tif (txq->nb_free < nb_used)\n+\t\t\tgoto end_of_tx;\n+\n+\t\ttx_offload.l2_len = tx_pkt->l2_len;\n+\t\ttx_offload.l3_len = tx_pkt->l3_len;\n+\t\ttx_offload.l4_len = tx_pkt->l4_len;\n+\t\ttx_offload.tso_segsz = tx_pkt->tso_segsz;\n+\n+\t\tfirst = tx_pkt;\n+\t\ttxd = &txr[tx_id];\n+\t\thlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ?\n+\t\t\t(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :\n+\t\t\ttx_pkt->pkt_len;\n+\n+\t\tsw_ring[sw_id] = tx_pkt;\n+\t\tif (!is_fifo_avail(txq, hlen)) {\n+\t\t\tgve_tx_clean(txq);\n+\t\t\tif (!is_fifo_avail(txq, hlen))\n+\t\t\t\tgoto end_of_tx;\n+\t\t}\n+\t\taddr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off;\n+\t\tfifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);\n+\n+\t\t/* For TSO, check if there's enough fifo space for data first */\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\tif (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen)) {\n+\t\t\t\tgve_tx_clean(txq);\n+\t\t\t\tif (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen))\n+\t\t\t\t\tgoto end_of_tx;\n+\t\t\t}\n+\t\t}\n+\t\tif (tx_pkt->nb_segs == 1 || ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\trte_memcpy((void *)(fifo_addr + txq->fifo_base), (void *)addr, hlen);\n+\t\telse\n+\t\t\trte_pktmbuf_read(tx_pkt, 0, hlen, (void *)(fifo_addr + txq->fifo_base));\n+\t\tgve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, fifo_addr);\n+\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\taddr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off + hlen;\n+\t\t\tfifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, tx_pkt->pkt_len - hlen);\n+\t\t\tif (tx_pkt->nb_segs == 1)\n+\t\t\t\trte_memcpy((void *)(fifo_addr + txq->fifo_base), (void *)addr,\n+\t\t\t\t\t tx_pkt->pkt_len - hlen);\n+\t\t\telse\n+\t\t\t\trte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,\n+\t\t\t\t\t\t (void *)(fifo_addr + txq->fifo_base));\n+\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t tx_pkt->pkt_len - hlen, fifo_addr);\n+\t\t}\n+\n+\t\t/* record mbuf in sw_ring for free */\n+\t\tfor (i = 1; i < first->nb_segs; i++) {\n+\t\t\tsw_id = (sw_id + 1) & mask;\n+\t\t\ttx_pkt = tx_pkt->next;\n+\t\t\tsw_ring[sw_id] = tx_pkt;\n+\t\t}\n+\n+\t\tsw_id = (sw_id + 1) & mask;\n+\t\ttx_id = (tx_id + 1) & mask;\n+\n+\t\ttxq->nb_free -= nb_used;\n+\t\ttxq->sw_nb_free -= first->nb_segs;\n+\t\ttx_tail += nb_used;\n+\t}\n+\n+end_of_tx:\n+\tif (nb_tx) {\n+\t\trte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);\n+\t\ttxq->tx_tail = tx_tail;\n+\t\ttxq->sw_tail = sw_id;\n+\t}\n+\n+\treturn nb_tx;\n+}\n+\n+static inline uint16_t\n+gve_tx_burst_ra(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tunion gve_tx_offload tx_offload = {0};\n+\tvolatile union gve_tx_desc *txr, *txd;\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\tstruct rte_mbuf **sw_ring = txq->sw_ring;\n+\tuint16_t mask = txq->nb_tx_desc - 1;\n+\tuint16_t tx_id = txq->tx_tail & mask;\n+\tuint32_t tx_tail = txq->tx_tail;\n+\tstruct rte_mbuf *tx_pkt, *first;\n+\tuint16_t nb_used, hlen, i;\n+\tuint64_t ol_flags, addr;\n+\tuint16_t nb_tx = 0;\n+\n+\ttxr = txq->tx_desc_ring;\n+\n+\tif (txq->nb_free < txq->free_thresh)\n+\t\tgve_tx_clean(txq);\n+\n+\tfor (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {\n+\t\ttx_pkt = *tx_pkts++;\n+\t\tol_flags = tx_pkt->ol_flags;\n+\n+\t\tnb_used = tx_pkt->nb_segs;\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG)\n+\t\t\tnb_used++;\n+\n+\t\tif (txq->nb_free < nb_used)\n+\t\t\tgoto end_of_tx;\n+\n+\t\ttx_offload.l2_len = tx_pkt->l2_len;\n+\t\ttx_offload.l3_len = tx_pkt->l3_len;\n+\t\ttx_offload.l4_len = tx_pkt->l4_len;\n+\t\ttx_offload.tso_segsz = tx_pkt->tso_segsz;\n+\n+\t\tfirst = tx_pkt;\n+\t\ttxd = &txr[tx_id];\n+\n+\t\thlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ?\n+\t\t\t(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :\n+\t\t\ttx_pkt->pkt_len;\n+\t\t/*\n+\t\t * if tso, the driver needs to fill 2 descs for 1 mbuf\n+\t\t * so only put this mbuf into the 1st tx entry in sw ring\n+\t\t */\n+\t\tsw_ring[tx_id] = tx_pkt;\n+\t\taddr = rte_mbuf_data_iova(tx_pkt);\n+\t\tgve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, addr);\n+\n+\t\tif (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\taddr = rte_mbuf_data_iova(tx_pkt) + hlen;\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t tx_pkt->data_len - hlen, addr);\n+\t\t}\n+\n+\t\tfor (i = 1; i < first->nb_segs; i++) {\n+\t\t\ttx_id = (tx_id + 1) & mask;\n+\t\t\ttxd = &txr[tx_id];\n+\t\t\ttx_pkt = tx_pkt->next;\n+\t\t\tsw_ring[tx_id] = tx_pkt;\n+\t\t\taddr = rte_mbuf_data_iova(tx_pkt);\n+\t\t\tgve_tx_fill_seg_desc(txd, ol_flags, tx_offload,\n+\t\t\t\t\t tx_pkt->data_len, addr);\n+\t\t}\n+\t\ttx_id = (tx_id + 1) & mask;\n+\n+\t\ttxq->nb_free -= nb_used;\n+\t\ttx_tail += nb_used;\n+\t}\n+\n+end_of_tx:\n+\tif (nb_tx) {\n+\t\trte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);\n+\t\ttxq->tx_tail = tx_tail;\n+\t}\n+\n+\treturn nb_tx;\n+}\n+\n+uint16_t\n+gve_tx_burst(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tstruct gve_tx_queue *txq = tx_queue;\n+\n+\tif (txq->is_gqi_qpl)\n+\t\treturn gve_tx_burst_qpl(tx_queue, tx_pkts, nb_pkts);\n+\n+\treturn gve_tx_burst_ra(tx_queue, tx_pkts, nb_pkts);\n+}\n+\n static inline void\n gve_reset_txq(struct gve_tx_queue *txq)\n {\n", "prefixes": [ "07/10" ] }