@@ -28,11 +28,11 @@
#define IAVF_RX_MAX_DATA_BUF_SIZE (16 * 1024 - 128)
/* used for Vector PMD */
-#define IAVF_VPMD_RX_BURST 32
+#define IAVF_VPMD_RX_BURST CI_VPMD_RX_BURST
#define IAVF_VPMD_TX_BURST 32
-#define IAVF_VPMD_RXQ_REARM_THRESH 32
-#define IAVF_VPMD_DESCS_PER_LOOP 4
-#define IAVF_VPMD_DESCS_PER_LOOP_WIDE 8
+#define IAVF_VPMD_RXQ_REARM_THRESH CI_VPMD_RX_REARM_THRESH
+#define IAVF_VPMD_DESCS_PER_LOOP CI_VPMD_DESCS_PER_LOOP
+#define IAVF_VPMD_DESCS_PER_LOOP_WIDE CI_VPMD_DESCS_PER_LOOP_WIDE
#define IAVF_VPMD_TX_MAX_FREE_BUF 64
#define IAVF_TX_NO_VECTOR_FLAGS ( \
@@ -2,6 +2,7 @@
* Copyright(c) 2019 Intel Corporation
*/
+#include "../common/rx_vec_x86.h"
#include "iavf_rxtx_vec_common.h"
#include <rte_vect.h>
@@ -9,7 +10,7 @@
static __rte_always_inline void
iavf_rxq_rearm(struct ci_rx_queue *rxq)
{
- iavf_rxq_rearm_common(rxq, false);
+ ci_rxq_rearm(rxq, CI_RX_VEC_LEVEL_AVX2);
}
#define PKTLEN_SHIFT 10
@@ -2,6 +2,7 @@
* Copyright(c) 2020 Intel Corporation
*/
+#include "../common/rx_vec_x86.h"
#include "iavf_rxtx_vec_common.h"
#include <rte_vect.h>
@@ -29,7 +30,7 @@
static __rte_always_inline void
iavf_rxq_rearm(struct ci_rx_queue *rxq)
{
- iavf_rxq_rearm_common(rxq, true);
+ ci_rxq_rearm(rxq, CI_RX_VEC_LEVEL_AVX512);
}
#define IAVF_RX_LEN_MASK 0x80808080
@@ -236,80 +236,4 @@ iavf_txd_enable_offload(__rte_unused struct rte_mbuf *tx_pkt,
*txd_hi |= ((uint64_t)td_cmd) << IAVF_TXD_QW1_CMD_SHIFT;
}
-
-#ifdef RTE_ARCH_X86
-static __rte_always_inline void
-iavf_rxq_rearm_common(struct ci_rx_queue *rxq, __rte_unused bool avx512)
-{
- int i;
- uint16_t rx_id;
- volatile union ci_rx_desc *rxdp;
- struct ci_rx_entry *rxp = &rxq->sw_ring[rxq->rxrearm_start];
-
- rxdp = rxq->rx_ring + rxq->rxrearm_start;
-
- /* Pull 'n' more MBUFs into the software ring */
- if (rte_mempool_get_bulk(rxq->mp,
- (void *)rxp,
- IAVF_VPMD_RXQ_REARM_THRESH) < 0) {
- if (rxq->rxrearm_nb + IAVF_VPMD_RXQ_REARM_THRESH >=
- rxq->nb_rx_desc) {
- __m128i dma_addr0;
-
- dma_addr0 = _mm_setzero_si128();
- for (i = 0; i < IAVF_VPMD_DESCS_PER_LOOP; i++) {
- rxp[i].mbuf = &rxq->fake_mbuf;
- _mm_store_si128(RTE_CAST_PTR(__m128i *, &rxdp[i].read),
- dma_addr0);
- }
- }
- rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
- IAVF_VPMD_RXQ_REARM_THRESH;
- return;
- }
-
- struct rte_mbuf *mb0, *mb1;
- __m128i dma_addr0, dma_addr1;
- __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
- RTE_PKTMBUF_HEADROOM);
- /* Initialize the mbufs in vector, process 2 mbufs in one loop */
- for (i = 0; i < IAVF_VPMD_RXQ_REARM_THRESH; i += 2, rxp += 2) {
- __m128i vaddr0, vaddr1;
-
- mb0 = rxp[0].mbuf;
- mb1 = rxp[1].mbuf;
-
- /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
- RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
- offsetof(struct rte_mbuf, buf_addr) + 8);
- vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
- vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
-
- /* convert pa to dma_addr hdr/data */
- dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
- dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
-
- /* add headroom to pa values */
- dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
- dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
-
- /* flush desc with pa dma_addr */
- _mm_store_si128(RTE_CAST_PTR(__m128i *, &rxdp++->read), dma_addr0);
- _mm_store_si128(RTE_CAST_PTR(__m128i *, &rxdp++->read), dma_addr1);
- }
-
- rxq->rxrearm_start += IAVF_VPMD_RXQ_REARM_THRESH;
- if (rxq->rxrearm_start >= rxq->nb_rx_desc)
- rxq->rxrearm_start = 0;
-
- rxq->rxrearm_nb -= IAVF_VPMD_RXQ_REARM_THRESH;
-
- rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
- (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
-
- /* Update the tail pointer on the NIC */
- IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
-}
-#endif
-
#endif
@@ -14,64 +14,12 @@
#include "iavf_rxtx.h"
#include "iavf_rxtx_vec_common.h"
+#include "../common/rx_vec_arm.h"
+
static inline void
iavf_rxq_rearm(struct ci_rx_queue *rxq)
{
- int i;
- uint16_t rx_id;
- volatile union ci_rx_desc *rxdp;
- struct ci_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
- struct rte_mbuf *mb0, *mb1;
- uint64x2_t dma_addr0, dma_addr1;
- uint64x2_t zero = vdupq_n_u64(0);
- uint64_t paddr;
-
- rxdp = rxq->rx_ring + rxq->rxrearm_start;
-
- /* Pull 'n' more MBUFs into the software ring */
- if (unlikely(rte_mempool_get_bulk(rxq->mp,
- (void *)rxep,
- IAVF_VPMD_RXQ_REARM_THRESH) < 0)) {
- if (rxq->rxrearm_nb + IAVF_VPMD_RXQ_REARM_THRESH >=
- rxq->nb_rx_desc) {
- for (i = 0; i < IAVF_VPMD_DESCS_PER_LOOP; i++) {
- rxep[i].mbuf = &rxq->fake_mbuf;
- vst1q_u64(RTE_CAST_PTR(uint64_t *, &rxdp[i].read), zero);
- }
- }
- rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
- IAVF_VPMD_RXQ_REARM_THRESH;
- return;
- }
-
- /* Initialize the mbufs in vector, process 2 mbufs in one loop */
- for (i = 0; i < IAVF_VPMD_RXQ_REARM_THRESH; i += 2, rxep += 2) {
- mb0 = rxep[0].mbuf;
- mb1 = rxep[1].mbuf;
-
- paddr = mb0->buf_iova + RTE_PKTMBUF_HEADROOM;
- dma_addr0 = vdupq_n_u64(paddr);
-
- /* flush desc with pa dma_addr */
- vst1q_u64(RTE_CAST_PTR(uint64_t *, &rxdp++->read), dma_addr0);
-
- paddr = mb1->buf_iova + RTE_PKTMBUF_HEADROOM;
- dma_addr1 = vdupq_n_u64(paddr);
- vst1q_u64(RTE_CAST_PTR(uint64_t *, &rxdp++->read), dma_addr1);
- }
-
- rxq->rxrearm_start += IAVF_VPMD_RXQ_REARM_THRESH;
- if (rxq->rxrearm_start >= rxq->nb_rx_desc)
- rxq->rxrearm_start = 0;
-
- rxq->rxrearm_nb -= IAVF_VPMD_RXQ_REARM_THRESH;
-
- rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
- (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
-
- rte_io_wmb();
- /* Update the tail pointer on the NIC */
- IAVF_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rx_id);
+ ci_rxq_rearm(rxq);
}
static inline void
@@ -9,82 +9,14 @@
#include "iavf.h"
#include "iavf_rxtx.h"
#include "iavf_rxtx_vec_common.h"
+#include "../common/rx_vec_x86.h"
#include <rte_vect.h>
static inline void
iavf_rxq_rearm(struct ci_rx_queue *rxq)
{
- int i;
- uint16_t rx_id;
-
- volatile union ci_rx_desc *rxdp;
- struct ci_rx_entry *rxp = &rxq->sw_ring[rxq->rxrearm_start];
- struct rte_mbuf *mb0, *mb1;
- __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
- RTE_PKTMBUF_HEADROOM);
- __m128i dma_addr0, dma_addr1;
-
- rxdp = rxq->rx_ring + rxq->rxrearm_start;
-
- /* Pull 'n' more MBUFs into the software ring */
- if (rte_mempool_get_bulk(rxq->mp, (void *)rxp,
- rxq->rx_free_thresh) < 0) {
- if (rxq->rxrearm_nb + rxq->rx_free_thresh >= rxq->nb_rx_desc) {
- dma_addr0 = _mm_setzero_si128();
- for (i = 0; i < IAVF_VPMD_DESCS_PER_LOOP; i++) {
- rxp[i].mbuf = &rxq->fake_mbuf;
- _mm_store_si128(RTE_CAST_PTR(__m128i *, &rxdp[i].read),
- dma_addr0);
- }
- }
- rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
- rxq->rx_free_thresh;
- return;
- }
-
- /* Initialize the mbufs in vector, process 2 mbufs in one loop */
- for (i = 0; i < rxq->rx_free_thresh; i += 2, rxp += 2) {
- __m128i vaddr0, vaddr1;
-
- mb0 = rxp[0].mbuf;
- mb1 = rxp[1].mbuf;
-
- /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
- RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
- offsetof(struct rte_mbuf, buf_addr) + 8);
- vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
- vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
-
- /* convert pa to dma_addr hdr/data */
- dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
- dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
-
- /* add headroom to pa values */
- dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
- dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
-
- /* flush desc with pa dma_addr */
- _mm_store_si128(RTE_CAST_PTR(__m128i *, &rxdp++->read), dma_addr0);
- _mm_store_si128(RTE_CAST_PTR(__m128i *, &rxdp++->read), dma_addr1);
- }
-
- rxq->rxrearm_start += rxq->rx_free_thresh;
- if (rxq->rxrearm_start >= rxq->nb_rx_desc)
- rxq->rxrearm_start = 0;
-
- rxq->rxrearm_nb -= rxq->rx_free_thresh;
-
- rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
- (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
-
- PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
- "rearm_start=%u rearm_nb=%u",
- rxq->port_id, rxq->queue_id,
- rx_id, rxq->rxrearm_start, rxq->rxrearm_nb);
-
- /* Update the tail pointer on the NIC */
- IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
+ ci_rxq_rearm(rxq, CI_RX_VEC_LEVEL_SSE);
}
static inline void