diff mbox series

[v3,06/13] net/ionic: memcpy descriptors when using Q-in-CMB

Message ID 20240207031317.32293-7-andrew.boyer@amd.com (mailing list archive)
State Accepted, archived
Delegated to: Ferruh Yigit
Headers
Series net/ionic: miscellaneous fixes and improvements |

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Boyer, Andrew Feb. 7, 2024, 3:13 a.m. UTC 
  From: Neel Patel <neel.patel@amd.com>

They can be batched together this way, reducing the number
of PCIe transactions. This improves transmit PPS by up to 50% in
some configurations.

Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
Signed-off-by: Neel Patel <neel.patel@amd.com>
---
 drivers/net/ionic/ionic_dev.c         |  9 +++--
 drivers/net/ionic/ionic_dev.h         |  6 ++-
 drivers/net/ionic/ionic_lif.c         | 26 +++++++++----
 drivers/net/ionic/ionic_rxtx.h        | 56 +++++++++++++++++++++++++++
 drivers/net/ionic/ionic_rxtx_sg.c     | 18 ++++-----
 drivers/net/ionic/ionic_rxtx_simple.c | 18 ++++-----
 6 files changed, 101 insertions(+), 32 deletions(-)
diff mbox series

Patch

diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c
index 70c14882ed..7f15914f74 100644
--- a/drivers/net/ionic/ionic_dev.c
+++ b/drivers/net/ionic/ionic_dev.c
@@ -369,17 +369,19 @@  ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs)
 	q->index = index;
 	q->num_descs = num_descs;
 	q->size_mask = num_descs - 1;
-	q->head_idx = 0;
-	q->tail_idx = 0;
+	ionic_q_reset(q);
 
 	return 0;
 }
 
 void
-ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa)
+ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa,
+			void *cmb_base, rte_iova_t cmb_base_pa)
 {
 	q->base = base;
 	q->base_pa = base_pa;
+	q->cmb_base = cmb_base;
+	q->cmb_base_pa = cmb_base_pa;
 }
 
 void
@@ -393,5 +395,6 @@  void
 ionic_q_reset(struct ionic_queue *q)
 {
 	q->head_idx = 0;
+	q->cmb_head_idx = 0;
 	q->tail_idx = 0;
 }
diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h
index 971c261b27..3a366247f1 100644
--- a/drivers/net/ionic/ionic_dev.h
+++ b/drivers/net/ionic/ionic_dev.h
@@ -145,11 +145,13 @@  struct ionic_queue {
 	uint16_t num_descs;
 	uint16_t num_segs;
 	uint16_t head_idx;
+	uint16_t cmb_head_idx;
 	uint16_t tail_idx;
 	uint16_t size_mask;
 	uint8_t type;
 	uint8_t hw_type;
 	void *base;
+	void *cmb_base;
 	void *sg_base;
 	struct ionic_doorbell __iomem *db;
 	void **info;
@@ -158,6 +160,7 @@  struct ionic_queue {
 	uint32_t hw_index;
 	rte_iova_t base_pa;
 	rte_iova_t sg_base_pa;
+	rte_iova_t cmb_base_pa;
 };
 
 #define IONIC_INTR_NONE		(-1)
@@ -244,7 +247,8 @@  uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do,
 
 int ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs);
 void ionic_q_reset(struct ionic_queue *q);
-void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa);
+void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa,
+				 void *cmb_base, rte_iova_t cmb_base_pa);
 void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa);
 
 static inline uint16_t
diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c
index fe2112c057..2713f8aa24 100644
--- a/drivers/net/ionic/ionic_lif.c
+++ b/drivers/net/ionic/ionic_lif.c
@@ -572,10 +572,11 @@  ionic_qcq_alloc(struct ionic_lif *lif,
 {
 	struct ionic_qcq *new;
 	uint32_t q_size, cq_size, sg_size, total_size;
-	void *q_base, *cq_base, *sg_base;
+	void *q_base, *cmb_q_base, *cq_base, *sg_base;
 	rte_iova_t q_base_pa = 0;
 	rte_iova_t cq_base_pa = 0;
 	rte_iova_t sg_base_pa = 0;
+	rte_iova_t cmb_q_base_pa = 0;
 	size_t page_size = rte_mem_page_size();
 	int err;
 
@@ -666,19 +667,22 @@  ionic_qcq_alloc(struct ionic_lif *lif,
 			IONIC_PRINT(ERR, "Cannot reserve queue from NIC mem");
 			return -ENOMEM;
 		}
-		q_base = (void *)
+		cmb_q_base = (void *)
 			((uintptr_t)lif->adapter->bars.bar[2].vaddr +
 			 (uintptr_t)lif->adapter->cmb_offset);
 		/* CMB PA is a relative address */
-		q_base_pa = lif->adapter->cmb_offset;
+		cmb_q_base_pa = lif->adapter->cmb_offset;
 		lif->adapter->cmb_offset += q_size;
+	} else {
+		cmb_q_base = NULL;
+		cmb_q_base_pa = 0;
 	}
 
 	IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 		"SG-base-PA = %#jx",
 		q_base_pa, cq_base_pa, sg_base_pa);
 
-	ionic_q_map(&new->q, q_base, q_base_pa);
+	ionic_q_map(&new->q, q_base, q_base_pa, cmb_q_base, cmb_q_base_pa);
 	ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 
 	*qcq = new;
@@ -1583,7 +1587,6 @@  ionic_lif_txq_init(struct ionic_tx_qcq *txq)
 			.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
 			.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
 			.ring_size = rte_log2_u32(q->num_descs),
-			.ring_base = rte_cpu_to_le_64(q->base_pa),
 			.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
 			.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
 		},
@@ -1592,8 +1595,12 @@  ionic_lif_txq_init(struct ionic_tx_qcq *txq)
 
 	if (txq->flags & IONIC_QCQ_F_SG)
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
-	if (txq->flags & IONIC_QCQ_F_CMB)
+	if (txq->flags & IONIC_QCQ_F_CMB) {
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->cmb_base_pa);
+	} else {
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->base_pa);
+	}
 
 	IONIC_PRINT(DEBUG, "txq_init.index %d", q->index);
 	IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "", q->base_pa);
@@ -1638,7 +1645,6 @@  ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
 			.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
 			.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
 			.ring_size = rte_log2_u32(q->num_descs),
-			.ring_base = rte_cpu_to_le_64(q->base_pa),
 			.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
 			.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
 		},
@@ -1647,8 +1653,12 @@  ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
 
 	if (rxq->flags & IONIC_QCQ_F_SG)
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
-	if (rxq->flags & IONIC_QCQ_F_CMB)
+	if (rxq->flags & IONIC_QCQ_F_CMB) {
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->cmb_base_pa);
+	} else {
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->base_pa);
+	}
 
 	IONIC_PRINT(DEBUG, "rxq_init.index %d", q->index);
 	IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "", q->base_pa);
diff --git a/drivers/net/ionic/ionic_rxtx.h b/drivers/net/ionic/ionic_rxtx.h
index 8537141597..5348395956 100644
--- a/drivers/net/ionic/ionic_rxtx.h
+++ b/drivers/net/ionic/ionic_rxtx.h
@@ -77,4 +77,60 @@  uint16_t ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 int ionic_rx_fill_sg(struct ionic_rx_qcq *rxq);
 
+static inline void
+ionic_rxq_flush(struct ionic_queue *q)
+{
+	struct ionic_rxq_desc *desc_base = q->base;
+	struct ionic_rxq_desc *cmb_desc_base = q->cmb_base;
+
+	if (q->cmb_base) {
+		if (q->head_idx < q->cmb_head_idx) {
+			/* copy [cmb_head, num_descs) */
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+				(void *)&desc_base[q->cmb_head_idx],
+				(q->num_descs - q->cmb_head_idx) * sizeof(*desc_base));
+			/* copy [0, head) */
+			rte_memcpy((void *)&cmb_desc_base[0],
+				(void *)&desc_base[0],
+				q->head_idx * sizeof(*desc_base));
+		} else {
+			/* copy [cmb_head, head) */
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+				(void *)&desc_base[q->cmb_head_idx],
+				(q->head_idx - q->cmb_head_idx) * sizeof(*desc_base));
+		}
+		q->cmb_head_idx = q->head_idx;
+	}
+
+	ionic_q_flush(q);
+}
+
+static inline void
+ionic_txq_flush(struct ionic_queue *q)
+{
+	struct ionic_txq_desc *desc_base = q->base;
+	struct ionic_txq_desc *cmb_desc_base = q->cmb_base;
+
+	if (q->cmb_base) {
+		if (q->head_idx < q->cmb_head_idx) {
+			/* copy [cmb_head, num_descs) */
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+				(void *)&desc_base[q->cmb_head_idx],
+				(q->num_descs - q->cmb_head_idx) * sizeof(*desc_base));
+			/* copy [0, head) */
+			rte_memcpy((void *)&cmb_desc_base[0],
+				(void *)&desc_base[0],
+				q->head_idx * sizeof(*desc_base));
+		} else {
+			/* copy [cmb_head, head) */
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+				(void *)&desc_base[q->cmb_head_idx],
+				(q->head_idx - q->cmb_head_idx) * sizeof(*desc_base));
+		}
+		q->cmb_head_idx = q->head_idx;
+	}
+
+	ionic_q_flush(q);
+}
+
 #endif /* _IONIC_RXTX_H_ */
diff --git a/drivers/net/ionic/ionic_rxtx_sg.c b/drivers/net/ionic/ionic_rxtx_sg.c
index 1392342463..92e1d6e259 100644
--- a/drivers/net/ionic/ionic_rxtx_sg.c
+++ b/drivers/net/ionic/ionic_rxtx_sg.c
@@ -166,6 +166,7 @@  ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 {
 	struct ionic_tx_qcq *txq = tx_queue;
 	struct ionic_queue *q = &txq->qcq.q;
+	struct ionic_txq_desc *desc_base = q->base;
 	struct ionic_tx_stats *stats = &txq->stats;
 	struct rte_mbuf *mbuf;
 	uint32_t bytes_tx = 0;
@@ -173,9 +174,7 @@  ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 	uint64_t then, now, hz, delta;
 	int err;
 
-	struct ionic_txq_desc *desc_base = q->base;
-	if (!(txq->flags & IONIC_QCQ_F_CMB))
-		rte_prefetch0(&desc_base[q->head_idx]);
+	rte_prefetch0(&desc_base[q->head_idx]);
 	rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx));
 
 	if (nb_pkts) {
@@ -196,8 +195,7 @@  ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 	while (nb_tx < nb_pkts) {
 		uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
-		if (!(txq->flags & IONIC_QCQ_F_CMB))
-			rte_prefetch0(&desc_base[next_idx]);
+		rte_prefetch0(&desc_base[next_idx]);
 		rte_prefetch0(IONIC_INFO_PTR(q, next_idx));
 
 		if (nb_tx + 1 < nb_pkts) {
@@ -222,7 +220,7 @@  ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 	if (nb_tx > 0) {
 		rte_wmb();
-		ionic_q_flush(q);
+		ionic_txq_flush(q);
 
 		txq->last_wdog_cycles = rte_get_timer_cycles();
 
@@ -458,8 +456,7 @@  ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 		/* Prefetch 4 x 16B comp */
 		rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
 		/* Prefetch 4 x 16B descriptors */
-		if (!(rxq->flags & IONIC_QCQ_F_CMB))
-			rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
+		rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
 
 		/* Clean one descriptor */
 		ionic_rx_clean_one_sg(rxq, cq_desc, rx_svc);
@@ -478,7 +475,8 @@  ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 
 	/* Update the queue indices and ring the doorbell */
 	if (work_done) {
-		ionic_q_flush(q);
+		ionic_rxq_flush(q);
+
 		rxq->last_wdog_cycles = rte_get_timer_cycles();
 		rxq->wdog_ms = IONIC_Q_WDOG_MS;
 	} else {
@@ -542,7 +540,7 @@  ionic_rx_fill_sg(struct ionic_rx_qcq *rxq)
 		q->head_idx = Q_NEXT_TO_POST(q, 1);
 	}
 
-	ionic_q_flush(q);
+	ionic_rxq_flush(q);
 
 	return err;
 }
diff --git a/drivers/net/ionic/ionic_rxtx_simple.c b/drivers/net/ionic/ionic_rxtx_simple.c
index 00152c885a..f12f66f40c 100644
--- a/drivers/net/ionic/ionic_rxtx_simple.c
+++ b/drivers/net/ionic/ionic_rxtx_simple.c
@@ -139,6 +139,7 @@  ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 {
 	struct ionic_tx_qcq *txq = tx_queue;
 	struct ionic_queue *q = &txq->qcq.q;
+	struct ionic_txq_desc *desc_base = q->base;
 	struct ionic_tx_stats *stats = &txq->stats;
 	struct rte_mbuf *mbuf;
 	uint32_t bytes_tx = 0;
@@ -146,9 +147,7 @@  ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	uint64_t then, now, hz, delta;
 	int err;
 
-	struct ionic_txq_desc *desc_base = q->base;
-	if (!(txq->flags & IONIC_QCQ_F_CMB))
-		rte_prefetch0(&desc_base[q->head_idx]);
+	rte_prefetch0(&desc_base[q->head_idx]);
 	rte_prefetch0(&q->info[q->head_idx]);
 
 	if (nb_pkts) {
@@ -169,8 +168,7 @@  ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 	while (nb_tx < nb_pkts) {
 		uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
-		if (!(txq->flags & IONIC_QCQ_F_CMB))
-			rte_prefetch0(&desc_base[next_idx]);
+		rte_prefetch0(&desc_base[next_idx]);
 		rte_prefetch0(&q->info[next_idx]);
 
 		if (nb_tx + 1 < nb_pkts) {
@@ -195,7 +193,7 @@  ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 	if (nb_tx > 0) {
 		rte_wmb();
-		ionic_q_flush(q);
+		ionic_txq_flush(q);
 
 		txq->last_wdog_cycles = rte_get_timer_cycles();
 
@@ -379,8 +377,7 @@  ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 		/* Prefetch 4 x 16B comp */
 		rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
 		/* Prefetch 4 x 16B descriptors */
-		if (!(rxq->flags & IONIC_QCQ_F_CMB))
-			rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
+		rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
 
 		/* Clean one descriptor */
 		ionic_rx_clean_one(rxq, cq_desc, rx_svc);
@@ -399,7 +396,8 @@  ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 
 	/* Update the queue indices and ring the doorbell */
 	if (work_done) {
-		ionic_q_flush(q);
+		ionic_rxq_flush(q);
+
 		rxq->last_wdog_cycles = rte_get_timer_cycles();
 		rxq->wdog_ms = IONIC_Q_WDOG_MS;
 	} else {
@@ -463,7 +461,7 @@  ionic_rx_fill(struct ionic_rx_qcq *rxq)
 		q->head_idx = Q_NEXT_TO_POST(q, 1);
 	}
 
-	ionic_q_flush(q);
+	ionic_rxq_flush(q);
 
 	return err;
 }