@@ -313,6 +313,9 @@ idpf_dev_start(struct rte_eth_dev *dev)
goto err_mtu;
}
+ idpf_set_rx_function(dev);
+ idpf_set_tx_function(dev);
+
if (idpf_vc_ena_dis_vport(vport, true)) {
PMD_DRV_LOG(ERR, "Failed to enable vport");
goto err_vport;
@@ -764,8 +767,11 @@ idpf_dev_init(struct rte_eth_dev *dev, void *init_params)
/* for secondary processes, we don't initialise any further as primary
* has already done this work.
*/
- if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ idpf_set_rx_function(dev);
+ idpf_set_tx_function(dev);
return ret;
+ }
dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
@@ -1268,3 +1268,674 @@ idpf_stop_queues(struct rte_eth_dev *dev)
PMD_DRV_LOG(WARNING, "Fail to stop Tx queue %d", i);
}
}
+
+static void
+idpf_split_rx_bufq_refill(struct idpf_rx_queue *rx_bufq)
+{
+ volatile struct virtchnl2_splitq_rx_buf_desc *rx_buf_ring;
+ volatile struct virtchnl2_splitq_rx_buf_desc *rx_buf_desc;
+ uint16_t nb_refill = rx_bufq->rx_free_thresh;
+ uint16_t nb_desc = rx_bufq->nb_rx_desc;
+ uint16_t next_avail = rx_bufq->rx_tail;
+ struct rte_mbuf *nmb[rx_bufq->rx_free_thresh];
+ struct rte_eth_dev *dev;
+ uint64_t dma_addr;
+ uint16_t delta;
+
+ if (rx_bufq->nb_rx_hold < rx_bufq->rx_free_thresh)
+ return;
+
+ rx_buf_ring =
+ (volatile struct virtchnl2_splitq_rx_buf_desc *)rx_bufq->rx_ring;
+ delta = nb_desc - next_avail;
+ if (unlikely(delta < nb_refill)) {
+ if (likely(!rte_pktmbuf_alloc_bulk(rx_bufq->mp, nmb, delta))) {
+ for (int i = 0; i < delta; i++) {
+ rx_buf_desc = &rx_buf_ring[next_avail + i];
+ rx_bufq->sw_ring[next_avail + i] = nmb[i];
+ dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb[i]));
+ rx_buf_desc->hdr_addr = 0;
+ rx_buf_desc->pkt_addr = dma_addr;
+ }
+ nb_refill -= delta;
+ next_avail = 0;
+ rx_bufq->nb_rx_hold -= delta;
+ } else {
+ dev = &rte_eth_devices[rx_bufq->port_id];
+ dev->data->rx_mbuf_alloc_failed += nb_desc - next_avail;
+ PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u queue_id=%u",
+ rx_bufq->port_id, rx_bufq->queue_id);
+ return;
+ }
+ }
+
+ if (nb_desc - next_avail >= nb_refill) {
+ if (likely(!rte_pktmbuf_alloc_bulk(rx_bufq->mp, nmb, nb_refill))) {
+ for (int i = 0; i < nb_refill; i++) {
+ rx_buf_desc = &rx_buf_ring[next_avail + i];
+ rx_bufq->sw_ring[next_avail + i] = nmb[i];
+ dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb[i]));
+ rx_buf_desc->hdr_addr = 0;
+ rx_buf_desc->pkt_addr = dma_addr;
+ }
+ next_avail += nb_refill;
+ rx_bufq->nb_rx_hold -= nb_refill;
+ } else {
+ dev = &rte_eth_devices[rx_bufq->port_id];
+ dev->data->rx_mbuf_alloc_failed += nb_desc - next_avail;
+ PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u queue_id=%u",
+ rx_bufq->port_id, rx_bufq->queue_id);
+ }
+ }
+
+ IDPF_PCI_REG_WRITE(rx_bufq->qrx_tail, next_avail);
+
+ rx_bufq->rx_tail = next_avail;
+}
+
+uint16_t
+idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc_ring;
+ volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc;
+ uint16_t pktlen_gen_bufq_id;
+ struct idpf_rx_queue *rxq;
+ const uint32_t *ptype_tbl;
+ struct rte_mbuf *rxm;
+ uint16_t rx_id_bufq1;
+ uint16_t rx_id_bufq2;
+ uint16_t pkt_len;
+ uint16_t bufq_id;
+ uint16_t gen_id;
+ uint16_t rx_id;
+ uint16_t nb_rx;
+
+ nb_rx = 0;
+ rxq = (struct idpf_rx_queue *)rx_queue;
+
+ if (unlikely(!rxq) || unlikely(!rxq->q_started))
+ return nb_rx;
+
+ rx_id = rxq->rx_tail;
+ rx_id_bufq1 = rxq->bufq1->rx_next_avail;
+ rx_id_bufq2 = rxq->bufq2->rx_next_avail;
+ rx_desc_ring =
+ (volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *)rxq->rx_ring;
+ ptype_tbl = rxq->adapter->ptype_tbl;
+
+ if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)
+ rxq->hw_register_set = 1;
+
+ while (nb_rx < nb_pkts) {
+ rx_desc = &rx_desc_ring[rx_id];
+
+ pktlen_gen_bufq_id =
+ rte_le_to_cpu_16(rx_desc->pktlen_gen_bufq_id);
+ gen_id = (pktlen_gen_bufq_id &
+ VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M) >>
+ VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_S;
+ if (gen_id != rxq->expected_gen_id)
+ break;
+
+ pkt_len = (pktlen_gen_bufq_id &
+ VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_M) >>
+ VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_S;
+ if (!pkt_len)
+ PMD_RX_LOG(ERR, "Packet length is 0");
+
+ rx_id++;
+ if (unlikely(rx_id == rxq->nb_rx_desc)) {
+ rx_id = 0;
+ rxq->expected_gen_id ^= 1;
+ }
+
+ bufq_id = (pktlen_gen_bufq_id &
+ VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_M) >>
+ VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_S;
+ if (!bufq_id) {
+ rxm = rxq->bufq1->sw_ring[rx_id_bufq1];
+ rx_id_bufq1++;
+ if (unlikely(rx_id_bufq1 == rxq->bufq1->nb_rx_desc))
+ rx_id_bufq1 = 0;
+ rxq->bufq1->nb_rx_hold++;
+ } else {
+ rxm = rxq->bufq2->sw_ring[rx_id_bufq2];
+ rx_id_bufq2++;
+ if (unlikely(rx_id_bufq2 == rxq->bufq2->nb_rx_desc))
+ rx_id_bufq2 = 0;
+ rxq->bufq2->nb_rx_hold++;
+ }
+
+ rxm->pkt_len = pkt_len;
+ rxm->data_len = pkt_len;
+ rxm->data_off = RTE_PKTMBUF_HEADROOM;
+ rxm->next = NULL;
+ rxm->nb_segs = 1;
+ rxm->port = rxq->port_id;
+ rxm->ol_flags = 0;
+ rxm->packet_type =
+ ptype_tbl[(rte_le_to_cpu_16(rx_desc->ptype_err_fflags0) &
+ VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_M) >>
+ VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_S];
+
+ rx_pkts[nb_rx++] = rxm;
+ }
+
+ if (nb_rx) {
+ rxq->rx_tail = rx_id;
+ if (rx_id_bufq1 != rxq->bufq1->rx_next_avail)
+ rxq->bufq1->rx_next_avail = rx_id_bufq1;
+ if (rx_id_bufq2 != rxq->bufq2->rx_next_avail)
+ rxq->bufq2->rx_next_avail = rx_id_bufq2;
+
+ idpf_split_rx_bufq_refill(rxq->bufq1);
+ idpf_split_rx_bufq_refill(rxq->bufq2);
+ }
+
+ return nb_rx;
+}
+
+static inline void
+idpf_split_tx_free(struct idpf_tx_queue *cq)
+{
+ volatile struct idpf_splitq_tx_compl_desc *compl_ring = cq->compl_ring;
+ volatile struct idpf_splitq_tx_compl_desc *txd;
+ uint16_t next = cq->tx_tail;
+ struct idpf_tx_entry *txe;
+ struct idpf_tx_queue *txq;
+ uint16_t gen, qid, q_head;
+ uint8_t ctype;
+
+ txd = &compl_ring[next];
+ gen = (rte_le_to_cpu_16(txd->qid_comptype_gen) &
+ IDPF_TXD_COMPLQ_GEN_M) >> IDPF_TXD_COMPLQ_GEN_S;
+ if (gen != cq->expected_gen_id)
+ return;
+
+ ctype = (rte_le_to_cpu_16(txd->qid_comptype_gen) &
+ IDPF_TXD_COMPLQ_COMPL_TYPE_M) >> IDPF_TXD_COMPLQ_COMPL_TYPE_S;
+ qid = (rte_le_to_cpu_16(txd->qid_comptype_gen) &
+ IDPF_TXD_COMPLQ_QID_M) >> IDPF_TXD_COMPLQ_QID_S;
+ q_head = rte_le_to_cpu_16(txd->q_head_compl_tag.compl_tag);
+ txq = cq->txqs[qid - cq->tx_start_qid];
+
+ switch (ctype) {
+ case IDPF_TXD_COMPLT_RE:
+ if (q_head == 0)
+ txq->last_desc_cleaned = txq->nb_tx_desc - 1;
+ else
+ txq->last_desc_cleaned = q_head - 1;
+ if (unlikely(!(txq->last_desc_cleaned % 32))) {
+ PMD_DRV_LOG(ERR, "unexpected desc (head = %u) completion.",
+ q_head);
+ return;
+ }
+
+ break;
+ case IDPF_TXD_COMPLT_RS:
+ txq->nb_free++;
+ txq->nb_used--;
+ txe = &txq->sw_ring[q_head];
+ if (txe->mbuf) {
+ rte_pktmbuf_free_seg(txe->mbuf);
+ txe->mbuf = NULL;
+ }
+ break;
+ default:
+ PMD_DRV_LOG(ERR, "unknown completion type.");
+ return;
+ }
+
+ if (++next == cq->nb_tx_desc) {
+ next = 0;
+ cq->expected_gen_id ^= 1;
+ }
+
+ cq->tx_tail = next;
+}
+
+uint16_t
+idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts)
+{
+ struct idpf_tx_queue *txq = (struct idpf_tx_queue *)tx_queue;
+ volatile struct idpf_flex_tx_sched_desc *txr;
+ volatile struct idpf_flex_tx_sched_desc *txd;
+ struct idpf_tx_entry *sw_ring;
+ struct idpf_tx_entry *txe, *txn;
+ uint16_t nb_used, tx_id, sw_id;
+ struct rte_mbuf *tx_pkt;
+ uint16_t nb_to_clean;
+ uint16_t nb_tx = 0;
+
+ if (unlikely(!txq) || unlikely(!txq->q_started))
+ return nb_tx;
+
+ txr = txq->desc_ring;
+ sw_ring = txq->sw_ring;
+ tx_id = txq->tx_tail;
+ sw_id = txq->sw_tail;
+ txe = &sw_ring[sw_id];
+
+ for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+ tx_pkt = tx_pkts[nb_tx];
+
+ if (txq->nb_free <= txq->free_thresh) {
+ /* TODO: Need to refine
+ * 1. free and clean: Better to decide a clean destination instead of
+ * loop times. And don't free mbuf when RS got immediately, free when
+ * transmit or according to the clean destination.
+ * Now, just ingnore the RE write back, free mbuf when get RS
+ * 2. out-of-order rewrite back haven't be supported, SW head and HW head
+ * need to be separated.
+ **/
+ nb_to_clean = 2 * txq->rs_thresh;
+ while (nb_to_clean--)
+ idpf_split_tx_free(txq->complq);
+ }
+
+ if (txq->nb_free < tx_pkt->nb_segs)
+ break;
+ nb_used = tx_pkt->nb_segs;
+
+
+ do {
+ txd = &txr[tx_id];
+ txn = &sw_ring[txe->next_id];
+ txe->mbuf = tx_pkt;
+
+ /* Setup TX descriptor */
+ txd->buf_addr =
+ rte_cpu_to_le_64(rte_mbuf_data_iova(tx_pkt));
+ txd->qw1.cmd_dtype =
+ rte_cpu_to_le_16(IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE);
+ txd->qw1.rxr_bufsize = tx_pkt->data_len;
+ txd->qw1.compl_tag = sw_id;
+ tx_id++;
+ if (tx_id == txq->nb_tx_desc)
+ tx_id = 0;
+ sw_id = txe->next_id;
+ txe = txn;
+ tx_pkt = tx_pkt->next;
+ } while (tx_pkt);
+
+ /* fill the last descriptor with End of Packet (EOP) bit */
+ txd->qw1.cmd_dtype |= IDPF_TXD_FLEX_FLOW_CMD_EOP;
+
+ if (unlikely(!(tx_id % 32)))
+ txd->qw1.cmd_dtype |= IDPF_TXD_FLEX_FLOW_CMD_RE;
+ txq->nb_free = (uint16_t)(txq->nb_free - nb_used);
+ txq->nb_used = (uint16_t)(txq->nb_used + nb_used);
+ }
+
+ /* update the tail pointer if any packets were processed */
+ if (likely(nb_tx)) {
+ IDPF_PCI_REG_WRITE(txq->qtx_tail, tx_id);
+ txq->tx_tail = tx_id;
+ txq->sw_tail = sw_id;
+ }
+
+ return nb_tx;
+}
+
+static inline void
+idpf_update_rx_tail(struct idpf_rx_queue *rxq, uint16_t nb_hold,
+ uint16_t rx_id)
+{
+ nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
+
+ if (nb_hold > rxq->rx_free_thresh) {
+ PMD_RX_LOG(DEBUG,
+ "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u",
+ rxq->port_id, rxq->queue_id, rx_id, nb_hold);
+ rx_id = (uint16_t)((rx_id == 0) ?
+ (rxq->nb_rx_desc - 1) : (rx_id - 1));
+ IDPF_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+ nb_hold = 0;
+ }
+ rxq->nb_rx_hold = nb_hold;
+}
+
+uint16_t
+idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ volatile union virtchnl2_rx_desc *rx_ring;
+ volatile union virtchnl2_rx_desc *rxdp;
+ union virtchnl2_rx_desc rxd;
+ struct idpf_rx_queue *rxq;
+ const uint32_t *ptype_tbl;
+ uint16_t rx_id, nb_hold;
+ struct rte_eth_dev *dev;
+ uint16_t rx_packet_len;
+ struct rte_mbuf *rxm;
+ struct rte_mbuf *nmb;
+ uint16_t rx_status0;
+ uint64_t dma_addr;
+ uint16_t nb_rx;
+
+ nb_rx = 0;
+ nb_hold = 0;
+ rxq = rx_queue;
+
+ if (unlikely(!rxq) || unlikely(!rxq->q_started))
+ return nb_rx;
+
+ rx_id = rxq->rx_tail;
+ rx_ring = rxq->rx_ring;
+ ptype_tbl = rxq->adapter->ptype_tbl;
+
+ if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)
+ rxq->hw_register_set = 1;
+
+ while (nb_rx < nb_pkts) {
+ rxdp = &rx_ring[rx_id];
+ rx_status0 = rte_le_to_cpu_16(rxdp->flex_nic_wb.status_error0);
+
+ /* Check the DD bit first */
+ if (!(rx_status0 & (1 << VIRTCHNL2_RX_FLEX_DESC_STATUS0_DD_S)))
+ break;
+
+ nmb = rte_mbuf_raw_alloc(rxq->mp);
+ if (unlikely(!nmb)) {
+ dev = &rte_eth_devices[rxq->port_id];
+ dev->data->rx_mbuf_alloc_failed++;
+ PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
+ "queue_id=%u", rxq->port_id, rxq->queue_id);
+ break;
+ }
+ rxd = *rxdp; /* copy descriptor in ring to temp variable*/
+
+ nb_hold++;
+ rxm = rxq->sw_ring[rx_id];
+ rxq->sw_ring[rx_id] = nmb;
+ rx_id++;
+ if (unlikely(rx_id == rxq->nb_rx_desc))
+ rx_id = 0;
+
+ /* Prefetch next mbuf */
+ rte_prefetch0(rxq->sw_ring[rx_id]);
+
+ /* When next RX descriptor is on a cache line boundary,
+ * prefetch the next 4 RX descriptors and next 8 pointers
+ * to mbufs.
+ */
+ if ((rx_id & 0x3) == 0) {
+ rte_prefetch0(&rx_ring[rx_id]);
+ rte_prefetch0(rxq->sw_ring[rx_id]);
+ }
+ dma_addr =
+ rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
+ rxdp->read.hdr_addr = 0;
+ rxdp->read.pkt_addr = dma_addr;
+
+ rx_packet_len = (rte_cpu_to_le_16(rxd.flex_nic_wb.pkt_len) &
+ VIRTCHNL2_RX_FLEX_DESC_PKT_LEN_M);
+
+ rxm->data_off = RTE_PKTMBUF_HEADROOM;
+ rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
+ rxm->nb_segs = 1;
+ rxm->next = NULL;
+ rxm->pkt_len = rx_packet_len;
+ rxm->data_len = rx_packet_len;
+ rxm->port = rxq->port_id;
+ rxm->ol_flags = 0;
+ rxm->packet_type =
+ ptype_tbl[(uint8_t)(rte_cpu_to_le_16(rxd.flex_nic_wb.ptype_flex_flags0) &
+ VIRTCHNL2_RX_FLEX_DESC_PTYPE_M)];
+
+
+ rx_pkts[nb_rx++] = rxm;
+ }
+ rxq->rx_tail = rx_id;
+
+ idpf_update_rx_tail(rxq, nb_hold, rx_id);
+
+ return nb_rx;
+}
+
+static inline int
+idpf_xmit_cleanup(struct idpf_tx_queue *txq)
+{
+ uint16_t last_desc_cleaned = txq->last_desc_cleaned;
+ struct idpf_tx_entry *sw_ring = txq->sw_ring;
+ uint16_t nb_tx_desc = txq->nb_tx_desc;
+ uint16_t desc_to_clean_to;
+ uint16_t nb_tx_to_clean;
+ uint16_t i;
+
+ volatile struct idpf_flex_tx_desc *txd = txq->tx_ring;
+
+ desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->rs_thresh);
+ if (desc_to_clean_to >= nb_tx_desc)
+ desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
+
+ desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
+ /* In the writeback Tx desccriptor, the only significant fields are the 4-bit DTYPE */
+ if ((txd[desc_to_clean_to].qw1.cmd_dtype &
+ rte_cpu_to_le_16(IDPF_TXD_QW1_DTYPE_M)) !=
+ rte_cpu_to_le_16(IDPF_TX_DESC_DTYPE_DESC_DONE)) {
+ PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done "
+ "(port=%d queue=%d)", desc_to_clean_to,
+ txq->port_id, txq->queue_id);
+ return -1;
+ }
+
+ if (last_desc_cleaned > desc_to_clean_to)
+ nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
+ desc_to_clean_to);
+ else
+ nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
+ last_desc_cleaned);
+
+ txd[desc_to_clean_to].qw1.cmd_dtype = 0;
+ txd[desc_to_clean_to].qw1.buf_size = 0;
+ for (i = 0; i < RTE_DIM(txd[desc_to_clean_to].qw1.flex.raw); i++)
+ txd[desc_to_clean_to].qw1.flex.raw[i] = 0;
+
+ txq->last_desc_cleaned = desc_to_clean_to;
+ txq->nb_free = (uint16_t)(txq->nb_free + nb_tx_to_clean);
+
+ return 0;
+}
+
+/* TX function */
+uint16_t
+idpf_singleq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts)
+{
+ volatile struct idpf_flex_tx_desc *txd;
+ volatile struct idpf_flex_tx_desc *txr;
+ struct idpf_tx_entry *txe, *txn;
+ struct idpf_tx_entry *sw_ring;
+ struct idpf_tx_queue *txq;
+ struct rte_mbuf *tx_pkt;
+ struct rte_mbuf *m_seg;
+ uint64_t buf_dma_addr;
+ uint16_t tx_last;
+ uint16_t nb_used;
+ uint16_t td_cmd;
+ uint16_t tx_id;
+ uint16_t nb_tx;
+ uint16_t slen;
+
+ nb_tx = 0;
+ txq = tx_queue;
+
+ if (unlikely(!txq) || unlikely(!txq->q_started))
+ return nb_tx;
+
+ sw_ring = txq->sw_ring;
+ txr = txq->tx_ring;
+ tx_id = txq->tx_tail;
+ txe = &sw_ring[tx_id];
+
+ /* Check if the descriptor ring needs to be cleaned. */
+ if (txq->nb_free < txq->free_thresh)
+ (void)idpf_xmit_cleanup(txq);
+
+ for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+ td_cmd = 0;
+
+ tx_pkt = *tx_pkts++;
+ RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf);
+
+ /* The number of descriptors that must be allocated for
+ * a packet equals to the number of the segments of that
+ * packet plus 1 context descriptor if needed.
+ */
+ nb_used = (uint16_t)(tx_pkt->nb_segs);
+ tx_last = (uint16_t)(tx_id + nb_used - 1);
+
+ /* Circular ring */
+ if (tx_last >= txq->nb_tx_desc)
+ tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
+
+ PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u"
+ " tx_first=%u tx_last=%u",
+ txq->port_id, txq->queue_id, tx_id, tx_last);
+
+ if (nb_used > txq->nb_free) {
+ if (idpf_xmit_cleanup(txq)) {
+ if (nb_tx == 0)
+ return 0;
+ goto end_of_tx;
+ }
+ if (unlikely(nb_used > txq->rs_thresh)) {
+ while (nb_used > txq->nb_free) {
+ if (idpf_xmit_cleanup(txq)) {
+ if (nb_tx == 0)
+ return 0;
+ goto end_of_tx;
+ }
+ }
+ }
+ }
+
+
+ m_seg = tx_pkt;
+ do {
+ txd = &txr[tx_id];
+ txn = &sw_ring[txe->next_id];
+
+ if (txe->mbuf)
+ rte_pktmbuf_free_seg(txe->mbuf);
+ txe->mbuf = m_seg;
+
+ /* Setup TX Descriptor */
+ slen = m_seg->data_len;
+ buf_dma_addr = rte_mbuf_data_iova(m_seg);
+ txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr);
+ txd->qw1.buf_size = slen;
+ txd->qw1.cmd_dtype = rte_cpu_to_le_16(IDPF_TX_DESC_DTYPE_FLEX_DATA <<
+ IDPF_FLEX_TXD_QW1_DTYPE_S);
+
+ txe->last_id = tx_last;
+ tx_id = txe->next_id;
+ txe = txn;
+ m_seg = m_seg->next;
+ } while (m_seg);
+
+ /* The last packet data descriptor needs End Of Packet (EOP) */
+ td_cmd |= IDPF_TX_FLEX_DESC_CMD_EOP;
+ txq->nb_used = (uint16_t)(txq->nb_used + nb_used);
+ txq->nb_free = (uint16_t)(txq->nb_free - nb_used);
+
+ if (txq->nb_used >= txq->rs_thresh) {
+ PMD_TX_LOG(DEBUG, "Setting RS bit on TXD id="
+ "%4u (port=%d queue=%d)",
+ tx_last, txq->port_id, txq->queue_id);
+
+ td_cmd |= IDPF_TX_FLEX_DESC_CMD_RS;
+
+ /* Update txq RS bit counters */
+ txq->nb_used = 0;
+ }
+
+ txd->qw1.cmd_dtype |= rte_cpu_to_le_16(td_cmd << IDPF_FLEX_TXD_QW1_CMD_S);
+ }
+
+end_of_tx:
+ rte_wmb();
+
+ PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
+ txq->port_id, txq->queue_id, tx_id, nb_tx);
+
+ IDPF_PCI_REG_WRITE(txq->qtx_tail, tx_id);
+ txq->tx_tail = tx_id;
+
+ return nb_tx;
+}
+
+/* TX prep functions */
+uint16_t
+idpf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts)
+{
+ int i, ret;
+ uint64_t ol_flags;
+ struct rte_mbuf *m;
+
+ for (i = 0; i < nb_pkts; i++) {
+ m = tx_pkts[i];
+ ol_flags = m->ol_flags;
+
+ /* Check condition for nb_segs > IDPF_TX_MAX_MTU_SEG. */
+ if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
+ if (m->nb_segs > IDPF_TX_MAX_MTU_SEG) {
+ rte_errno = EINVAL;
+ return i;
+ }
+ } else if ((m->tso_segsz < IDPF_MIN_TSO_MSS) ||
+ (m->tso_segsz > IDPF_MAX_TSO_MSS) ||
+ (m->pkt_len > IDPF_MAX_TSO_FRAME_SIZE)) {
+ /* MSS outside the range are considered malicious */
+ rte_errno = EINVAL;
+ return i;
+ }
+
+ if (m->pkt_len < IDPF_MIN_FRAME_SIZE) {
+ rte_errno = EINVAL;
+ return i;
+ }
+
+ ret = rte_net_intel_cksum_prepare(m);
+ if (ret != 0) {
+ rte_errno = -ret;
+ return i;
+ }
+ }
+
+ return i;
+}
+
+void
+idpf_set_rx_function(struct rte_eth_dev *dev)
+{
+ struct idpf_vport *vport = dev->data->dev_private;
+
+ if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) {
+ dev->rx_pkt_burst = idpf_splitq_recv_pkts;
+ return;
+ } else {
+ dev->rx_pkt_burst = idpf_singleq_recv_pkts;
+ return;
+ }
+}
+
+void
+idpf_set_tx_function(struct rte_eth_dev *dev)
+{
+ struct idpf_vport *vport = dev->data->dev_private;
+
+ if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) {
+ dev->tx_pkt_burst = idpf_splitq_xmit_pkts;
+ dev->tx_pkt_prepare = idpf_prep_pkts;
+ return;
+ } else {
+ dev->tx_pkt_burst = idpf_singleq_xmit_pkts;
+ dev->tx_pkt_prepare = idpf_prep_pkts;
+ return;
+ }
+}
@@ -44,9 +44,6 @@
#define IDPF_MAX_TSO_FRAME_SIZE 262143
#define IDPF_TX_MAX_MTU_SEG 10
-#define IDPF_TX_OFFLOAD_NOTSUP_MASK \
- (RTE_MBUF_F_TX_OFFLOAD_MASK ^ IDPF_TX_OFFLOAD_MASK)
-
#define IDPF_GET_PTYPE_SIZE(p) \
(sizeof(struct virtchnl2_ptype) + \
(((p)->proto_id_count ? ((p)->proto_id_count - 1) : 0) * sizeof((p)->proto_id[0])))
@@ -191,8 +188,21 @@ int idpf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id);
int idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);
void idpf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
+uint16_t idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+uint16_t idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+uint16_t idpf_singleq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+uint16_t idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+uint16_t idpf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
void idpf_stop_queues(struct rte_eth_dev *dev);
+void idpf_set_rx_function(struct rte_eth_dev *dev);
+void idpf_set_tx_function(struct rte_eth_dev *dev);
+
const uint32_t *idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev);
#endif /* _IDPF_RXTX_H_ */