@@ -3731,25 +3731,41 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
}
if (dev->data->scattered_rx) {
- if (!use_avx512) {
+ if (!use_avx2 && !use_avx512) {
PMD_DRV_LOG(DEBUG,
- "Using %sVector Scattered Rx (port %d).",
- use_avx2 ? "avx2 " : "",
+ "Using Vector Scattered Rx (port %d).",
dev->data->port_id);
} else {
- if (check_ret == IAVF_VECTOR_PATH)
- PMD_DRV_LOG(DEBUG,
- "Using AVX512 Vector Scattered Rx (port %d).",
- dev->data->port_id);
- else
- PMD_DRV_LOG(DEBUG,
- "Using AVX512 OFFLOAD Vector Scattered Rx (port %d).",
- dev->data->port_id);
+ if (use_avx2) {
+ if (check_ret == IAVF_VECTOR_PATH)
+ PMD_DRV_LOG(DEBUG,
+ "Using AVX2 Vector Scattered Rx (port %d).",
+ dev->data->port_id);
+ else
+ PMD_DRV_LOG(DEBUG,
+ "Using AVX2 OFFLOAD Vector Scattered Rx (port %d).",
+ dev->data->port_id);
+ } else {
+ if (check_ret == IAVF_VECTOR_PATH)
+ PMD_DRV_LOG(DEBUG,
+ "Using AVX512 Vector Scattered Rx (port %d).",
+ dev->data->port_id);
+ else
+ PMD_DRV_LOG(DEBUG,
+ "Using AVX512 OFFLOAD Vector Scattered Rx (port %d).",
+ dev->data->port_id);
+ }
}
if (use_flex) {
- dev->rx_pkt_burst = use_avx2 ?
- iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
- iavf_recv_scattered_pkts_vec_flex_rxd;
+ dev->rx_pkt_burst = iavf_recv_scattered_pkts_vec_flex_rxd;
+ if (use_avx2) {
+ if (check_ret == IAVF_VECTOR_PATH)
+ dev->rx_pkt_burst =
+ iavf_recv_scattered_pkts_vec_avx2_flex_rxd;
+ else
+ dev->rx_pkt_burst =
+ iavf_recv_scattered_pkts_vec_avx2_flex_rxd_offload;
+ }
#ifdef CC_AVX512_SUPPORT
if (use_avx512) {
if (check_ret == IAVF_VECTOR_PATH)
@@ -3761,9 +3777,15 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
}
#endif
} else {
- dev->rx_pkt_burst = use_avx2 ?
- iavf_recv_scattered_pkts_vec_avx2 :
- iavf_recv_scattered_pkts_vec;
+ dev->rx_pkt_burst = iavf_recv_scattered_pkts_vec;
+ if (use_avx2) {
+ if (check_ret == IAVF_VECTOR_PATH)
+ dev->rx_pkt_burst =
+ iavf_recv_scattered_pkts_vec_avx2;
+ else
+ dev->rx_pkt_burst =
+ iavf_recv_scattered_pkts_vec_avx2_offload;
+ }
#ifdef CC_AVX512_SUPPORT
if (use_avx512) {
if (check_ret == IAVF_VECTOR_PATH)
@@ -3776,24 +3798,40 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
#endif
}
} else {
- if (!use_avx512) {
- PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).",
- use_avx2 ? "avx2 " : "",
+ if (!use_avx2 && !use_avx512) {
+ PMD_DRV_LOG(DEBUG, "Using Vector Rx (port %d).",
dev->data->port_id);
} else {
- if (check_ret == IAVF_VECTOR_PATH)
- PMD_DRV_LOG(DEBUG,
- "Using AVX512 Vector Rx (port %d).",
- dev->data->port_id);
- else
- PMD_DRV_LOG(DEBUG,
- "Using AVX512 OFFLOAD Vector Rx (port %d).",
- dev->data->port_id);
+ if (use_avx2) {
+ if (check_ret == IAVF_VECTOR_PATH)
+ PMD_DRV_LOG(DEBUG,
+ "Using AVX2 Vector Rx (port %d).",
+ dev->data->port_id);
+ else
+ PMD_DRV_LOG(DEBUG,
+ "Using AVX2 OFFLOAD Vector Rx (port %d).",
+ dev->data->port_id);
+ } else {
+ if (check_ret == IAVF_VECTOR_PATH)
+ PMD_DRV_LOG(DEBUG,
+ "Using AVX512 Vector Rx (port %d).",
+ dev->data->port_id);
+ else
+ PMD_DRV_LOG(DEBUG,
+ "Using AVX512 OFFLOAD Vector Rx (port %d).",
+ dev->data->port_id);
+ }
}
if (use_flex) {
- dev->rx_pkt_burst = use_avx2 ?
- iavf_recv_pkts_vec_avx2_flex_rxd :
- iavf_recv_pkts_vec_flex_rxd;
+ dev->rx_pkt_burst = iavf_recv_pkts_vec_flex_rxd;
+ if (use_avx2) {
+ if (check_ret == IAVF_VECTOR_PATH)
+ dev->rx_pkt_burst =
+ iavf_recv_pkts_vec_avx2_flex_rxd;
+ else
+ dev->rx_pkt_burst =
+ iavf_recv_pkts_vec_avx2_flex_rxd_offload;
+ }
#ifdef CC_AVX512_SUPPORT
if (use_avx512) {
if (check_ret == IAVF_VECTOR_PATH)
@@ -3805,9 +3843,15 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
}
#endif
} else {
- dev->rx_pkt_burst = use_avx2 ?
- iavf_recv_pkts_vec_avx2 :
- iavf_recv_pkts_vec;
+ dev->rx_pkt_burst = iavf_recv_pkts_vec;
+ if (use_avx2) {
+ if (check_ret == IAVF_VECTOR_PATH)
+ dev->rx_pkt_burst =
+ iavf_recv_pkts_vec_avx2;
+ else
+ dev->rx_pkt_burst =
+ iavf_recv_pkts_vec_avx2_offload;
+ }
#ifdef CC_AVX512_SUPPORT
if (use_avx512) {
if (check_ret == IAVF_VECTOR_PATH)
@@ -680,15 +680,26 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_avx2_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue,
struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_avx2_flex_rxd_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
@@ -18,10 +18,11 @@ iavf_rxq_rearm(struct iavf_rx_queue *rxq)
#define PKTLEN_SHIFT 10
-static inline uint16_t
+static __rte_always_inline uint16_t
_iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,
struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts, uint8_t *split_packet)
+ uint16_t nb_pkts, uint8_t *split_packet,
+ bool offload)
{
#define IAVF_DESCS_PER_LOOP_AVX 8
@@ -332,30 +333,34 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,
*/
__m256i status0_7 = _mm256_unpacklo_epi64(status4_7,
status0_3);
+ __m256i mbuf_flags = _mm256_set1_epi32(0);
+
+ if (offload) {
+ /* now do flag manipulation */
+
+ /* get only flag/error bits we want */
+ const __m256i flag_bits =
+ _mm256_and_si256(status0_7, flags_mask);
+ /* set vlan and rss flags */
+ const __m256i vlan_flags =
+ _mm256_shuffle_epi8(vlan_flags_shuf, flag_bits);
+ const __m256i rss_flags =
+ _mm256_shuffle_epi8(rss_flags_shuf,
+ _mm256_srli_epi32(flag_bits, 11));
+ /**
+ * l3_l4_error flags, shuffle, then shift to correct adjustment
+ * of flags in flags_shuf, and finally mask out extra bits
+ */
+ __m256i l3_l4_flags = _mm256_shuffle_epi8(l3_l4_flags_shuf,
+ _mm256_srli_epi32(flag_bits, 22));
+ l3_l4_flags = _mm256_slli_epi32(l3_l4_flags, 1);
+ l3_l4_flags = _mm256_and_si256(l3_l4_flags, cksum_mask);
+
+ /* merge flags */
+ mbuf_flags = _mm256_or_si256(l3_l4_flags,
+ _mm256_or_si256(rss_flags, vlan_flags));
+ }
- /* now do flag manipulation */
-
- /* get only flag/error bits we want */
- const __m256i flag_bits =
- _mm256_and_si256(status0_7, flags_mask);
- /* set vlan and rss flags */
- const __m256i vlan_flags =
- _mm256_shuffle_epi8(vlan_flags_shuf, flag_bits);
- const __m256i rss_flags =
- _mm256_shuffle_epi8(rss_flags_shuf,
- _mm256_srli_epi32(flag_bits, 11));
- /**
- * l3_l4_error flags, shuffle, then shift to correct adjustment
- * of flags in flags_shuf, and finally mask out extra bits
- */
- __m256i l3_l4_flags = _mm256_shuffle_epi8(l3_l4_flags_shuf,
- _mm256_srli_epi32(flag_bits, 22));
- l3_l4_flags = _mm256_slli_epi32(l3_l4_flags, 1);
- l3_l4_flags = _mm256_and_si256(l3_l4_flags, cksum_mask);
-
- /* merge flags */
- const __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
- _mm256_or_si256(rss_flags, vlan_flags));
/**
* At this point, we have the 8 sets of flags in the low 16-bits
* of each 32-bit value in vlan0.
@@ -517,10 +522,11 @@ flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7)
return fdir_flags;
}
-static inline uint16_t
+static __rte_always_inline uint16_t
_iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts, uint8_t *split_packet)
+ uint16_t nb_pkts, uint8_t *split_packet,
+ bool offload)
{
#define IAVF_DESCS_PER_LOOP_AVX 8
@@ -808,6 +814,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
mb6_7 = _mm256_add_epi16(mb6_7, crc_adjust);
mb4_5 = _mm256_add_epi16(mb4_5, crc_adjust);
+
/**
* to get packet types, ptype is located in bit16-25
* of each 128bits
@@ -868,50 +875,52 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
*/
__m256i status0_7 = _mm256_unpacklo_epi64(status4_7,
status0_3);
-
- /* now do flag manipulation */
-
- /* get only flag/error bits we want */
- const __m256i flag_bits =
- _mm256_and_si256(status0_7, flags_mask);
- /**
- * l3_l4_error flags, shuffle, then shift to correct adjustment
- * of flags in flags_shuf, and finally mask out extra bits
- */
- __m256i l3_l4_flags = _mm256_shuffle_epi8(l3_l4_flags_shuf,
- _mm256_srli_epi32(flag_bits, 4));
- l3_l4_flags = _mm256_slli_epi32(l3_l4_flags, 1);
- __m256i l4_outer_mask = _mm256_set1_epi32(0x6);
- __m256i l4_outer_flags =
- _mm256_and_si256(l3_l4_flags, l4_outer_mask);
- l4_outer_flags = _mm256_slli_epi32(l4_outer_flags, 20);
-
- __m256i l3_l4_mask = _mm256_set1_epi32(~0x6);
-
- l3_l4_flags = _mm256_and_si256(l3_l4_flags, l3_l4_mask);
- l3_l4_flags = _mm256_or_si256(l3_l4_flags, l4_outer_flags);
- l3_l4_flags = _mm256_and_si256(l3_l4_flags, cksum_mask);
-
- /* set rss and vlan flags */
- const __m256i rss_vlan_flag_bits =
- _mm256_srli_epi32(flag_bits, 12);
- const __m256i rss_flags =
- _mm256_shuffle_epi8(rss_flags_shuf,
- rss_vlan_flag_bits);
-
+ __m256i mbuf_flags = _mm256_set1_epi32(0);
__m256i vlan_flags = _mm256_setzero_si256();
- if (rxq->rx_flags == IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG1)
- vlan_flags =
- _mm256_shuffle_epi8(vlan_flags_shuf,
+ if (offload) {
+ /* now do flag manipulation */
+
+ /* get only flag/error bits we want */
+ const __m256i flag_bits =
+ _mm256_and_si256(status0_7, flags_mask);
+ /**
+ * l3_l4_error flags, shuffle, then shift to correct adjustment
+ * of flags in flags_shuf, and finally mask out extra bits
+ */
+ __m256i l3_l4_flags = _mm256_shuffle_epi8(l3_l4_flags_shuf,
+ _mm256_srli_epi32(flag_bits, 4));
+ l3_l4_flags = _mm256_slli_epi32(l3_l4_flags, 1);
+ __m256i l4_outer_mask = _mm256_set1_epi32(0x6);
+ __m256i l4_outer_flags =
+ _mm256_and_si256(l3_l4_flags, l4_outer_mask);
+ l4_outer_flags = _mm256_slli_epi32(l4_outer_flags, 20);
+
+ __m256i l3_l4_mask = _mm256_set1_epi32(~0x6);
+
+ l3_l4_flags = _mm256_and_si256(l3_l4_flags, l3_l4_mask);
+ l3_l4_flags = _mm256_or_si256(l3_l4_flags, l4_outer_flags);
+ l3_l4_flags = _mm256_and_si256(l3_l4_flags, cksum_mask);
+
+ /* set rss and vlan flags */
+ const __m256i rss_vlan_flag_bits =
+ _mm256_srli_epi32(flag_bits, 12);
+ const __m256i rss_flags =
+ _mm256_shuffle_epi8(rss_flags_shuf,
rss_vlan_flag_bits);
- const __m256i rss_vlan_flags =
- _mm256_or_si256(rss_flags, vlan_flags);
+ if (rxq->rx_flags == IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG1)
+ vlan_flags =
+ _mm256_shuffle_epi8(vlan_flags_shuf,
+ rss_vlan_flag_bits);
- /* merge flags */
- __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
- rss_vlan_flags);
+ const __m256i rss_vlan_flags =
+ _mm256_or_si256(rss_flags, vlan_flags);
+
+ /* merge flags */
+ mbuf_flags = _mm256_or_si256(l3_l4_flags,
+ rss_vlan_flags);
+ }
if (rxq->fdir_enabled) {
const __m256i fdir_id4_7 =
@@ -955,176 +964,178 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
_mm256_extract_epi32(fdir_id0_7, 4);
} /* if() on fdir_enabled */
+ if (offload) {
#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
- /**
- * needs to load 2nd 16B of each desc for RSS hash parsing,
- * will cause performance drop to get into this context.
- */
- if (offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH ||
- rxq->rx_flags & IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG2_2) {
- /* load bottom half of every 32B desc */
- const __m128i raw_desc_bh7 =
- _mm_load_si128
- ((void *)(&rxdp[7].wb.status_error1));
- rte_compiler_barrier();
- const __m128i raw_desc_bh6 =
- _mm_load_si128
- ((void *)(&rxdp[6].wb.status_error1));
- rte_compiler_barrier();
- const __m128i raw_desc_bh5 =
- _mm_load_si128
- ((void *)(&rxdp[5].wb.status_error1));
- rte_compiler_barrier();
- const __m128i raw_desc_bh4 =
- _mm_load_si128
- ((void *)(&rxdp[4].wb.status_error1));
- rte_compiler_barrier();
- const __m128i raw_desc_bh3 =
- _mm_load_si128
- ((void *)(&rxdp[3].wb.status_error1));
- rte_compiler_barrier();
- const __m128i raw_desc_bh2 =
- _mm_load_si128
- ((void *)(&rxdp[2].wb.status_error1));
- rte_compiler_barrier();
- const __m128i raw_desc_bh1 =
- _mm_load_si128
- ((void *)(&rxdp[1].wb.status_error1));
- rte_compiler_barrier();
- const __m128i raw_desc_bh0 =
- _mm_load_si128
- ((void *)(&rxdp[0].wb.status_error1));
-
- __m256i raw_desc_bh6_7 =
- _mm256_inserti128_si256
- (_mm256_castsi128_si256(raw_desc_bh6),
- raw_desc_bh7, 1);
- __m256i raw_desc_bh4_5 =
- _mm256_inserti128_si256
- (_mm256_castsi128_si256(raw_desc_bh4),
- raw_desc_bh5, 1);
- __m256i raw_desc_bh2_3 =
- _mm256_inserti128_si256
- (_mm256_castsi128_si256(raw_desc_bh2),
- raw_desc_bh3, 1);
- __m256i raw_desc_bh0_1 =
- _mm256_inserti128_si256
- (_mm256_castsi128_si256(raw_desc_bh0),
- raw_desc_bh1, 1);
-
- if (offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH) {
- /**
- * to shift the 32b RSS hash value to the
- * highest 32b of each 128b before mask
- */
- __m256i rss_hash6_7 =
- _mm256_slli_epi64(raw_desc_bh6_7, 32);
- __m256i rss_hash4_5 =
- _mm256_slli_epi64(raw_desc_bh4_5, 32);
- __m256i rss_hash2_3 =
- _mm256_slli_epi64(raw_desc_bh2_3, 32);
- __m256i rss_hash0_1 =
- _mm256_slli_epi64(raw_desc_bh0_1, 32);
-
- const __m256i rss_hash_msk =
- _mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
- 0xFFFFFFFF, 0, 0, 0);
-
- rss_hash6_7 = _mm256_and_si256
- (rss_hash6_7, rss_hash_msk);
- rss_hash4_5 = _mm256_and_si256
- (rss_hash4_5, rss_hash_msk);
- rss_hash2_3 = _mm256_and_si256
- (rss_hash2_3, rss_hash_msk);
- rss_hash0_1 = _mm256_and_si256
- (rss_hash0_1, rss_hash_msk);
-
- mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
- mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
- mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
- mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
- }
-
- if (rxq->rx_flags & IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG2_2) {
- /* merge the status/error-1 bits into one register */
- const __m256i status1_4_7 =
- _mm256_unpacklo_epi32(raw_desc_bh6_7,
- raw_desc_bh4_5);
- const __m256i status1_0_3 =
- _mm256_unpacklo_epi32(raw_desc_bh2_3,
- raw_desc_bh0_1);
-
- const __m256i status1_0_7 =
- _mm256_unpacklo_epi64(status1_4_7,
- status1_0_3);
-
- const __m256i l2tag2p_flag_mask =
- _mm256_set1_epi32
- (1 << IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S);
-
- __m256i l2tag2p_flag_bits =
- _mm256_and_si256
- (status1_0_7, l2tag2p_flag_mask);
-
- l2tag2p_flag_bits =
- _mm256_srli_epi32(l2tag2p_flag_bits,
- IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S);
-
- const __m256i l2tag2_flags_shuf =
- _mm256_set_epi8(0, 0, 0, 0,
- 0, 0, 0, 0,
- 0, 0, 0, 0,
- 0, 0,
- RTE_MBUF_F_RX_VLAN |
- RTE_MBUF_F_RX_VLAN_STRIPPED,
- 0,
- /* end up 128-bits */
- 0, 0, 0, 0,
- 0, 0, 0, 0,
- 0, 0, 0, 0,
- 0, 0,
- RTE_MBUF_F_RX_VLAN |
- RTE_MBUF_F_RX_VLAN_STRIPPED,
- 0);
-
- vlan_flags =
- _mm256_shuffle_epi8(l2tag2_flags_shuf,
- l2tag2p_flag_bits);
-
- /* merge with vlan_flags */
- mbuf_flags = _mm256_or_si256
- (mbuf_flags, vlan_flags);
-
- /* L2TAG2_2 */
- __m256i vlan_tci6_7 =
- _mm256_slli_si256(raw_desc_bh6_7, 4);
- __m256i vlan_tci4_5 =
- _mm256_slli_si256(raw_desc_bh4_5, 4);
- __m256i vlan_tci2_3 =
- _mm256_slli_si256(raw_desc_bh2_3, 4);
- __m256i vlan_tci0_1 =
- _mm256_slli_si256(raw_desc_bh0_1, 4);
-
- const __m256i vlan_tci_msk =
- _mm256_set_epi32(0, 0xFFFF0000, 0, 0,
- 0, 0xFFFF0000, 0, 0);
-
- vlan_tci6_7 = _mm256_and_si256
- (vlan_tci6_7, vlan_tci_msk);
- vlan_tci4_5 = _mm256_and_si256
- (vlan_tci4_5, vlan_tci_msk);
- vlan_tci2_3 = _mm256_and_si256
- (vlan_tci2_3, vlan_tci_msk);
- vlan_tci0_1 = _mm256_and_si256
- (vlan_tci0_1, vlan_tci_msk);
-
- mb6_7 = _mm256_or_si256(mb6_7, vlan_tci6_7);
- mb4_5 = _mm256_or_si256(mb4_5, vlan_tci4_5);
- mb2_3 = _mm256_or_si256(mb2_3, vlan_tci2_3);
- mb0_1 = _mm256_or_si256(mb0_1, vlan_tci0_1);
- }
- } /* if() on RSS hash parsing */
+ /**
+ * needs to load 2nd 16B of each desc for RSS hash parsing,
+ * will cause performance drop to get into this context.
+ */
+ if (offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH ||
+ rxq->rx_flags & IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG2_2) {
+ /* load bottom half of every 32B desc */
+ const __m128i raw_desc_bh7 =
+ _mm_load_si128
+ ((void *)(&rxdp[7].wb.status_error1));
+ rte_compiler_barrier();
+ const __m128i raw_desc_bh6 =
+ _mm_load_si128
+ ((void *)(&rxdp[6].wb.status_error1));
+ rte_compiler_barrier();
+ const __m128i raw_desc_bh5 =
+ _mm_load_si128
+ ((void *)(&rxdp[5].wb.status_error1));
+ rte_compiler_barrier();
+ const __m128i raw_desc_bh4 =
+ _mm_load_si128
+ ((void *)(&rxdp[4].wb.status_error1));
+ rte_compiler_barrier();
+ const __m128i raw_desc_bh3 =
+ _mm_load_si128
+ ((void *)(&rxdp[3].wb.status_error1));
+ rte_compiler_barrier();
+ const __m128i raw_desc_bh2 =
+ _mm_load_si128
+ ((void *)(&rxdp[2].wb.status_error1));
+ rte_compiler_barrier();
+ const __m128i raw_desc_bh1 =
+ _mm_load_si128
+ ((void *)(&rxdp[1].wb.status_error1));
+ rte_compiler_barrier();
+ const __m128i raw_desc_bh0 =
+ _mm_load_si128
+ ((void *)(&rxdp[0].wb.status_error1));
+
+ __m256i raw_desc_bh6_7 =
+ _mm256_inserti128_si256
+ (_mm256_castsi128_si256(raw_desc_bh6),
+ raw_desc_bh7, 1);
+ __m256i raw_desc_bh4_5 =
+ _mm256_inserti128_si256
+ (_mm256_castsi128_si256(raw_desc_bh4),
+ raw_desc_bh5, 1);
+ __m256i raw_desc_bh2_3 =
+ _mm256_inserti128_si256
+ (_mm256_castsi128_si256(raw_desc_bh2),
+ raw_desc_bh3, 1);
+ __m256i raw_desc_bh0_1 =
+ _mm256_inserti128_si256
+ (_mm256_castsi128_si256(raw_desc_bh0),
+ raw_desc_bh1, 1);
+
+ if (offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH) {
+ /**
+ * to shift the 32b RSS hash value to the
+ * highest 32b of each 128b before mask
+ */
+ __m256i rss_hash6_7 =
+ _mm256_slli_epi64(raw_desc_bh6_7, 32);
+ __m256i rss_hash4_5 =
+ _mm256_slli_epi64(raw_desc_bh4_5, 32);
+ __m256i rss_hash2_3 =
+ _mm256_slli_epi64(raw_desc_bh2_3, 32);
+ __m256i rss_hash0_1 =
+ _mm256_slli_epi64(raw_desc_bh0_1, 32);
+
+ const __m256i rss_hash_msk =
+ _mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
+ 0xFFFFFFFF, 0, 0, 0);
+
+ rss_hash6_7 = _mm256_and_si256
+ (rss_hash6_7, rss_hash_msk);
+ rss_hash4_5 = _mm256_and_si256
+ (rss_hash4_5, rss_hash_msk);
+ rss_hash2_3 = _mm256_and_si256
+ (rss_hash2_3, rss_hash_msk);
+ rss_hash0_1 = _mm256_and_si256
+ (rss_hash0_1, rss_hash_msk);
+
+ mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
+ mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
+ mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
+ mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
+ }
+
+ if (rxq->rx_flags & IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG2_2) {
+ /* merge the status/error-1 bits into one register */
+ const __m256i status1_4_7 =
+ _mm256_unpacklo_epi32(raw_desc_bh6_7,
+ raw_desc_bh4_5);
+ const __m256i status1_0_3 =
+ _mm256_unpacklo_epi32(raw_desc_bh2_3,
+ raw_desc_bh0_1);
+
+ const __m256i status1_0_7 =
+ _mm256_unpacklo_epi64(status1_4_7,
+ status1_0_3);
+
+ const __m256i l2tag2p_flag_mask =
+ _mm256_set1_epi32
+ (1 << IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S);
+
+ __m256i l2tag2p_flag_bits =
+ _mm256_and_si256
+ (status1_0_7, l2tag2p_flag_mask);
+
+ l2tag2p_flag_bits =
+ _mm256_srli_epi32(l2tag2p_flag_bits,
+ IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S);
+
+ const __m256i l2tag2_flags_shuf =
+ _mm256_set_epi8(0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0,
+ RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0,
+ /* end up 128-bits */
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0,
+ RTE_MBUF_F_RX_VLAN |
+ RTE_MBUF_F_RX_VLAN_STRIPPED,
+ 0);
+
+ vlan_flags =
+ _mm256_shuffle_epi8(l2tag2_flags_shuf,
+ l2tag2p_flag_bits);
+
+ /* merge with vlan_flags */
+ mbuf_flags = _mm256_or_si256
+ (mbuf_flags, vlan_flags);
+
+ /* L2TAG2_2 */
+ __m256i vlan_tci6_7 =
+ _mm256_slli_si256(raw_desc_bh6_7, 4);
+ __m256i vlan_tci4_5 =
+ _mm256_slli_si256(raw_desc_bh4_5, 4);
+ __m256i vlan_tci2_3 =
+ _mm256_slli_si256(raw_desc_bh2_3, 4);
+ __m256i vlan_tci0_1 =
+ _mm256_slli_si256(raw_desc_bh0_1, 4);
+
+ const __m256i vlan_tci_msk =
+ _mm256_set_epi32(0, 0xFFFF0000, 0, 0,
+ 0, 0xFFFF0000, 0, 0);
+
+ vlan_tci6_7 = _mm256_and_si256
+ (vlan_tci6_7, vlan_tci_msk);
+ vlan_tci4_5 = _mm256_and_si256
+ (vlan_tci4_5, vlan_tci_msk);
+ vlan_tci2_3 = _mm256_and_si256
+ (vlan_tci2_3, vlan_tci_msk);
+ vlan_tci0_1 = _mm256_and_si256
+ (vlan_tci0_1, vlan_tci_msk);
+
+ mb6_7 = _mm256_or_si256(mb6_7, vlan_tci6_7);
+ mb4_5 = _mm256_or_si256(mb4_5, vlan_tci4_5);
+ mb2_3 = _mm256_or_si256(mb2_3, vlan_tci2_3);
+ mb0_1 = _mm256_or_si256(mb0_1, vlan_tci0_1);
+ }
+ } /* if() on RSS hash parsing */
#endif
+ }
/**
* At this point, we have the 8 sets of flags in the low 16-bits
@@ -1277,7 +1288,16 @@ uint16_t
iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
- return _iavf_recv_raw_pkts_vec_avx2(rx_queue, rx_pkts, nb_pkts, NULL);
+ return _iavf_recv_raw_pkts_vec_avx2(rx_queue, rx_pkts, nb_pkts,
+ NULL, false);
+}
+
+uint16_t
+iavf_recv_pkts_vec_avx2_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ return _iavf_recv_raw_pkts_vec_avx2(rx_queue, rx_pkts, nb_pkts,
+ NULL, true);
}
/**
@@ -1289,7 +1309,15 @@ iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
return _iavf_recv_raw_pkts_vec_avx2_flex_rxd(rx_queue, rx_pkts,
- nb_pkts, NULL);
+ nb_pkts, NULL, false);
+}
+
+uint16_t
+iavf_recv_pkts_vec_avx2_flex_rxd_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ return _iavf_recv_raw_pkts_vec_avx2_flex_rxd(rx_queue, rx_pkts,
+ nb_pkts, NULL, true);
}
/**
@@ -1297,16 +1325,16 @@ iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
* Notice:
* - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet
*/
-static uint16_t
+static __rte_always_inline uint16_t
iavf_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts)
+ uint16_t nb_pkts, bool offload)
{
struct iavf_rx_queue *rxq = rx_queue;
uint8_t split_flags[IAVF_VPMD_RX_MAX_BURST] = {0};
/* get some new buffers */
uint16_t nb_bufs = _iavf_recv_raw_pkts_vec_avx2(rxq, rx_pkts, nb_pkts,
- split_flags);
+ split_flags, offload);
if (nb_bufs == 0)
return 0;
@@ -1339,22 +1367,44 @@ iavf_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
* Notice:
* - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet
*/
-uint16_t
-iavf_recv_scattered_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts)
+static __rte_always_inline uint16_t
+iavf_recv_scattered_pkts_vec_avx2_common(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool offload)
{
uint16_t retval = 0;
while (nb_pkts > IAVF_VPMD_RX_MAX_BURST) {
uint16_t burst = iavf_recv_scattered_burst_vec_avx2(rx_queue,
- rx_pkts + retval, IAVF_VPMD_RX_MAX_BURST);
+ rx_pkts + retval, IAVF_VPMD_RX_MAX_BURST, offload);
retval += burst;
nb_pkts -= burst;
if (burst < IAVF_VPMD_RX_MAX_BURST)
return retval;
}
return retval + iavf_recv_scattered_burst_vec_avx2(rx_queue,
- rx_pkts + retval, nb_pkts);
+ rx_pkts + retval, nb_pkts, offload);
+}
+
+uint16_t
+iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ return iavf_recv_scattered_pkts_vec_avx2_common(rx_queue,
+ rx_pkts,
+ nb_pkts,
+ false);
+}
+
+uint16_t
+iavf_recv_scattered_pkts_vec_avx2_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ return iavf_recv_scattered_pkts_vec_avx2_common(rx_queue,
+ rx_pkts,
+ nb_pkts,
+ true);
}
/**
@@ -1363,17 +1413,17 @@ iavf_recv_scattered_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
* Notice:
* - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet
*/
-static uint16_t
+static __rte_always_inline uint16_t
iavf_recv_scattered_burst_vec_avx2_flex_rxd(void *rx_queue,
struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts)
+ uint16_t nb_pkts, bool offload)
{
struct iavf_rx_queue *rxq = rx_queue;
uint8_t split_flags[IAVF_VPMD_RX_MAX_BURST] = {0};
/* get some new buffers */
uint16_t nb_bufs = _iavf_recv_raw_pkts_vec_avx2_flex_rxd(rxq,
- rx_pkts, nb_pkts, split_flags);
+ rx_pkts, nb_pkts, split_flags, offload);
if (nb_bufs == 0)
return 0;
@@ -1406,26 +1456,50 @@ iavf_recv_scattered_burst_vec_avx2_flex_rxd(void *rx_queue,
* Notice:
* - nb_pkts < IAVF_DESCS_PER_LOOP, just return no packet
*/
-uint16_t
-iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
- struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts)
+static __rte_always_inline uint16_t
+iavf_recv_scattered_pkts_vec_avx2_flex_rxd_common(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, bool offload)
{
uint16_t retval = 0;
while (nb_pkts > IAVF_VPMD_RX_MAX_BURST) {
uint16_t burst =
iavf_recv_scattered_burst_vec_avx2_flex_rxd
- (rx_queue, rx_pkts + retval, IAVF_VPMD_RX_MAX_BURST);
+ (rx_queue, rx_pkts + retval, IAVF_VPMD_RX_MAX_BURST,
+ offload);
retval += burst;
nb_pkts -= burst;
if (burst < IAVF_VPMD_RX_MAX_BURST)
return retval;
}
return retval + iavf_recv_scattered_burst_vec_avx2_flex_rxd(rx_queue,
- rx_pkts + retval, nb_pkts);
+ rx_pkts + retval, nb_pkts, offload);
}
+uint16_t
+iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ return iavf_recv_scattered_pkts_vec_avx2_flex_rxd_common(rx_queue,
+ rx_pkts,
+ nb_pkts,
+ false);
+}
+
+uint16_t
+iavf_recv_scattered_pkts_vec_avx2_flex_rxd_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ return iavf_recv_scattered_pkts_vec_avx2_flex_rxd_common(rx_queue,
+ rx_pkts,
+ nb_pkts,
+ true);
+}
+
+
static __rte_always_inline void
iavf_vtx1(volatile struct iavf_tx_desc *txdp,
struct rte_mbuf *pkt, uint64_t flags, bool offload)