[dpdk-dev,06/14] fm10k: add Vector RX function
Commit Message
From: "Chen Jing D(Mark)" <jing.d.chen@intel.com>
Add func fm10k_recv_raw_pkts_vec to parse raw packets, in which
includes possible chained packets.
Add func fm10k_recv_pkts_vec to receive single mbuf packet.
Signed-off-by: Chen Jing D(Mark) <jing.d.chen@intel.com>
---
drivers/net/fm10k/fm10k.h | 1 +
drivers/net/fm10k/fm10k_rxtx_vec.c | 213 ++++++++++++++++++++++++++++++++++++
2 files changed, 214 insertions(+), 0 deletions(-)
Comments
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Chen Jing D(Mark)
> Sent: Tuesday, September 29, 2015 2:04 PM
> To: dev@dpdk.org
> Subject: [dpdk-dev] [PATCH 06/14] fm10k: add Vector RX function
>
> From: "Chen Jing D(Mark)" <jing.d.chen@intel.com>
>
> Add func fm10k_recv_raw_pkts_vec to parse raw packets, in which
> includes possible chained packets.
> Add func fm10k_recv_pkts_vec to receive single mbuf packet.
>
> Signed-off-by: Chen Jing D(Mark) <jing.d.chen@intel.com>
> ---
> drivers/net/fm10k/fm10k.h | 1 +
> drivers/net/fm10k/fm10k_rxtx_vec.c | 213 ++++++++++++++++++++++++++++++++++++
> 2 files changed, 214 insertions(+), 0 deletions(-)
>
> diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
> index d924cae..285254f 100644
> --- a/drivers/net/fm10k/fm10k.h
> +++ b/drivers/net/fm10k/fm10k.h
> @@ -327,4 +327,5 @@ uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
> uint16_t nb_pkts);
>
> int fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq);
> +uint16_t fm10k_recv_pkts_vec(void *, struct rte_mbuf **, uint16_t);
> #endif
> diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
> index 581a309..63b34b5 100644
> --- a/drivers/net/fm10k/fm10k_rxtx_vec.c
> +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
> @@ -281,3 +281,216 @@ fm10k_rxq_rearm(struct fm10k_rx_queue *rxq)
> /* Update the tail pointer on the NIC */
> FM10K_PCI_REG_WRITE(rxq->tail_ptr, rx_id);
> }
> +
> +/*
> + * vPMD receive routine, now only accept (nb_pkts == RTE_IXGBE_VPMD_RX_BURST)
> + * in one loop
> + *
> + * Notice:
> + * - nb_pkts < RTE_IXGBE_VPMD_RX_BURST, just return no packet
> + * - nb_pkts > RTE_IXGBE_VPMD_RX_BURST, only scan RTE_IXGBE_VPMD_RX_BURST
> + * numbers of DD bit
> + * - don't support ol_flags for rss and csum err
> + */
> +static inline uint16_t
> +fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
> + uint16_t nb_pkts, uint8_t *split_packet)
> +{
> + volatile union fm10k_rx_desc *rxdp;
> + struct rte_mbuf **mbufp;
> + uint16_t nb_pkts_recd;
> + int pos;
> + struct fm10k_rx_queue *rxq = rx_queue;
> + uint64_t var;
> + __m128i shuf_msk;
> + __m128i dd_check, eop_check;
> + uint16_t next_dd;
> +
> + next_dd = rxq->next_dd;
> +
> + if (unlikely(nb_pkts < RTE_FM10K_MAX_RX_BURST))
> + return 0;
> +
> + /* Just the act of getting into the function from the application is
> + * going to cost about 7 cycles
> + */
> + rxdp = rxq->hw_ring + next_dd;
> +
> + _mm_prefetch((const void *)rxdp, _MM_HINT_T0);
> +
> + /* See if we need to rearm the RX queue - gives the prefetch a bit
> + * of time to act
> + */
> + if (rxq->rxrearm_nb > RTE_FM10K_RXQ_REARM_THRESH)
> + fm10k_rxq_rearm(rxq);
> +
> + /* Before we start moving massive data around, check to see if
> + * there is actually a packet available
> + */
> + if (!(rxdp->d.staterr & FM10K_RXD_STATUS_DD))
> + return 0;
> +
> + /* 4 packets DD mask */
> + dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL);
> +
> + /* 4 packets EOP mask */
> + eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL);
> +
> + /* mask to shuffle from desc. to mbuf */
> + shuf_msk = _mm_set_epi8(
> + 7, 6, 5, 4, /* octet 4~7, 32bits rss */
> + 15, 14, /* octet 14~15, low 16 bits vlan_macip */
> + 13, 12, /* octet 12~13, 16 bits data_len */
> + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
> + 13, 12, /* octet 12~13, low 16 bits pkt_len */
> + 0xFF, 0xFF, /* skip high 16 bits pkt_type */
> + 0xFF, 0xFF /* Skip pkt_type field in shuffle operation */
> + );
> +
> + /* Cache is empty -> need to scan the buffer rings, but first move
> + * the next 'n' mbufs into the cache
> + */
> + mbufp = &rxq->sw_ring[next_dd];
> +
> + /* A. load 4 packet in one loop
> + * [A*. mask out 4 unused dirty field in desc]
> + * B. copy 4 mbuf point from swring to rx_pkts
> + * C. calc the number of DD bits among the 4 packets
> + * [C*. extract the end-of-packet bit, if requested]
> + * D. fill info. from desc to mbuf
> + */
> + for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
> + pos += RTE_FM10K_DESCS_PER_LOOP,
> + rxdp += RTE_FM10K_DESCS_PER_LOOP) {
> + __m128i descs0[RTE_FM10K_DESCS_PER_LOOP];
> + __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
> + __m128i zero, staterr, sterr_tmp1, sterr_tmp2;
> + __m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */
> +
> + if (split_packet) {
> + rte_prefetch0(&rx_pkts[pos]->cacheline1);
> + rte_prefetch0(&rx_pkts[pos + 1]->cacheline1);
> + rte_prefetch0(&rx_pkts[pos + 2]->cacheline1);
> + rte_prefetch0(&rx_pkts[pos + 3]->cacheline1);
> + }
Same thing as with i40e vPMD:
You are pretching junk addreses here.
Check out Zoltan's patch:
http://dpdk.org/dev/patchwork/patch/7190/
and related conversation:
http://dpdk.org/ml/archives/dev/2015-September/023715.html
I think there is the same issue here.
Konstantin
> +
> + /* B.1 load 1 mbuf point */
> + mbp1 = _mm_loadu_si128((__m128i *)&mbufp[pos]);
> +
> + /* Read desc statuses backwards to avoid race condition */
> + /* A.1 load 4 pkts desc */
> + descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
> +
> + /* B.2 copy 2 mbuf point into rx_pkts */
> + _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
> +
> + /* B.1 load 1 mbuf point */
> + mbp2 = _mm_loadu_si128((__m128i *)&mbufp[pos+2]);
> +
> + descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
> + /* B.1 load 2 mbuf point */
> + descs0[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
> + descs0[0] = _mm_loadu_si128((__m128i *)(rxdp));
> +
> + /* B.2 copy 2 mbuf point into rx_pkts */
> + _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2);
> +
> + /* avoid compiler reorder optimization */
> + rte_compiler_barrier();
> +
> + /* D.1 pkt 3,4 convert format from desc to pktmbuf */
> + pkt_mb4 = _mm_shuffle_epi8(descs0[3], shuf_msk);
> + pkt_mb3 = _mm_shuffle_epi8(descs0[2], shuf_msk);
> +
> + /* C.1 4=>2 filter staterr info only */
> + sterr_tmp2 = _mm_unpackhi_epi32(descs0[3], descs0[2]);
> + /* C.1 4=>2 filter staterr info only */
> + sterr_tmp1 = _mm_unpackhi_epi32(descs0[1], descs0[0]);
> +
> + /* set ol_flags with vlan packet type */
> + fm10k_desc_to_olflags_v(descs0, &rx_pkts[pos]);
> +
> + /* D.1 pkt 1,2 convert format from desc to pktmbuf */
> + pkt_mb2 = _mm_shuffle_epi8(descs0[1], shuf_msk);
> + pkt_mb1 = _mm_shuffle_epi8(descs0[0], shuf_msk);
> +
> + /* C.2 get 4 pkts staterr value */
> + zero = _mm_xor_si128(dd_check, dd_check);
> + staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
> +
> + /* D.3 copy final 3,4 data to rx_pkts */
> + _mm_storeu_si128((void *)&rx_pkts[pos+3]->rx_descriptor_fields1,
> + pkt_mb4);
> + _mm_storeu_si128((void *)&rx_pkts[pos+2]->rx_descriptor_fields1,
> + pkt_mb3);
> +
> + /* C* extract and record EOP bit */
> + if (split_packet) {
> + __m128i eop_shuf_mask = _mm_set_epi8(
> + 0xFF, 0xFF, 0xFF, 0xFF,
> + 0xFF, 0xFF, 0xFF, 0xFF,
> + 0xFF, 0xFF, 0xFF, 0xFF,
> + 0x04, 0x0C, 0x00, 0x08
> + );
> +
> + /* and with mask to extract bits, flipping 1-0 */
> + __m128i eop_bits = _mm_andnot_si128(staterr, eop_check);
> + /* the staterr values are not in order, as the count
> + * count of dd bits doesn't care. However, for end of
> + * packet tracking, we do care, so shuffle. This also
> + * compresses the 32-bit values to 8-bit
> + */
> + eop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask);
> + /* store the resulting 32-bit value */
> + *(int *)split_packet = _mm_cvtsi128_si32(eop_bits);
> + split_packet += RTE_FM10K_DESCS_PER_LOOP;
> +
> + /* zero-out next pointers */
> + rx_pkts[pos]->next = NULL;
> + rx_pkts[pos + 1]->next = NULL;
> + rx_pkts[pos + 2]->next = NULL;
> + rx_pkts[pos + 3]->next = NULL;
> + }
> +
> + /* C.3 calc available number of desc */
> + staterr = _mm_and_si128(staterr, dd_check);
> + staterr = _mm_packs_epi32(staterr, zero);
> +
> + /* D.3 copy final 1,2 data to rx_pkts */
> + _mm_storeu_si128((void *)&rx_pkts[pos+1]->rx_descriptor_fields1,
> + pkt_mb2);
> + _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
> + pkt_mb1);
> +
> + fm10k_desc_to_pktype_v(descs0, &rx_pkts[pos]);
> +
> + /* C.4 calc avaialbe number of desc */
> + var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
> + nb_pkts_recd += var;
> + if (likely(var != RTE_FM10K_DESCS_PER_LOOP))
> + break;
> + }
> +
> + /* Update our internal tail pointer */
> + rxq->next_dd = (uint16_t)(rxq->next_dd + nb_pkts_recd);
> + rxq->next_dd = (uint16_t)(rxq->next_dd & (rxq->nb_desc - 1));
> + rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
> +
> + return nb_pkts_recd;
> +}
> +
> +/* vPMD receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP)
> + *
> + * Notice:
> + * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet
> + * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
> + * numbers of DD bit
> + * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
> + * - don't support ol_flags for rss and csum err
> + */
> +uint16_t
> +fm10k_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
> + uint16_t nb_pkts)
> +{
> + return fm10k_recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
> +}
> --
> 1.7.7.6
On Tue, Sep 29, 2015 at 01:14:26PM +0000, Ananyev, Konstantin wrote:
>
>
> > -----Original Message-----
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Chen Jing D(Mark)
> > Sent: Tuesday, September 29, 2015 2:04 PM
> > To: dev@dpdk.org
> > Subject: [dpdk-dev] [PATCH 06/14] fm10k: add Vector RX function
> >
> > From: "Chen Jing D(Mark)" <jing.d.chen@intel.com>
> >
> > Add func fm10k_recv_raw_pkts_vec to parse raw packets, in which
> > includes possible chained packets.
> > Add func fm10k_recv_pkts_vec to receive single mbuf packet.
> >
> > Signed-off-by: Chen Jing D(Mark) <jing.d.chen@intel.com>
> > ---
> > drivers/net/fm10k/fm10k.h | 1 +
> > drivers/net/fm10k/fm10k_rxtx_vec.c | 213 ++++++++++++++++++++++++++++++++++++
> > 2 files changed, 214 insertions(+), 0 deletions(-)
> >
> > diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
> > index d924cae..285254f 100644
> > --- a/drivers/net/fm10k/fm10k.h
> > +++ b/drivers/net/fm10k/fm10k.h
> > @@ -327,4 +327,5 @@ uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
> > uint16_t nb_pkts);
> >
> > int fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq);
> > +uint16_t fm10k_recv_pkts_vec(void *, struct rte_mbuf **, uint16_t);
> > #endif
> > diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
> > index 581a309..63b34b5 100644
> > --- a/drivers/net/fm10k/fm10k_rxtx_vec.c
> > +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
> > @@ -281,3 +281,216 @@ fm10k_rxq_rearm(struct fm10k_rx_queue *rxq)
> > /* Update the tail pointer on the NIC */
> > FM10K_PCI_REG_WRITE(rxq->tail_ptr, rx_id);
> > }
> > +
> > +/*
> > + * vPMD receive routine, now only accept (nb_pkts == RTE_IXGBE_VPMD_RX_BURST)
> > + * in one loop
> > + *
> > + * Notice:
> > + * - nb_pkts < RTE_IXGBE_VPMD_RX_BURST, just return no packet
Why this limitation? I believe this limitation has already been removed for
ixgbe, so the same solution should be applicable here
/Bruce
> -----Original Message-----
> From: Ananyev, Konstantin
> Sent: Tuesday, September 29, 2015 9:14 PM
> To: Chen, Jing D; dev@dpdk.org
> Subject: RE: [dpdk-dev] [PATCH 06/14] fm10k: add Vector RX function
>
>
> > + /* A. load 4 packet in one loop
> > + * [A*. mask out 4 unused dirty field in desc]
> > + * B. copy 4 mbuf point from swring to rx_pkts
> > + * C. calc the number of DD bits among the 4 packets
> > + * [C*. extract the end-of-packet bit, if requested]
> > + * D. fill info. from desc to mbuf
> > + */
> > + for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
> > + pos += RTE_FM10K_DESCS_PER_LOOP,
> > + rxdp += RTE_FM10K_DESCS_PER_LOOP) {
> > + __m128i descs0[RTE_FM10K_DESCS_PER_LOOP];
> > + __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
> > + __m128i zero, staterr, sterr_tmp1, sterr_tmp2;
> > + __m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg.
> */
> > +
> > + if (split_packet) {
> > + rte_prefetch0(&rx_pkts[pos]->cacheline1);
> > + rte_prefetch0(&rx_pkts[pos + 1]->cacheline1);
> > + rte_prefetch0(&rx_pkts[pos + 2]->cacheline1);
> > + rte_prefetch0(&rx_pkts[pos + 3]->cacheline1);
> > + }
>
>
> Same thing as with i40e vPMD:
> You are pretching junk addreses here.
> Check out Zoltan's patch:
> http://dpdk.org/dev/patchwork/patch/7190/
> and related conversation:
> http://dpdk.org/ml/archives/dev/2015-September/023715.html
> I think there is the same issue here.
> Konstantin
>
Thanks for the comments, Konstantin! I'll check the material you referred to.
Hi, Bruce,
> -----Original Message-----
> From: Richardson, Bruce
> Sent: Tuesday, September 29, 2015 10:23 PM
> To: Ananyev, Konstantin
> Cc: Chen, Jing D; dev@dpdk.org
> Subject: Re: [dpdk-dev] [PATCH 06/14] fm10k: add Vector RX function
>
> On Tue, Sep 29, 2015 at 01:14:26PM +0000, Ananyev, Konstantin wrote:
> >
> >
> > > -----Original Message-----
> > > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Chen Jing
> > > D(Mark)
> > > Sent: Tuesday, September 29, 2015 2:04 PM
> > > To: dev@dpdk.org
> > > Subject: [dpdk-dev] [PATCH 06/14] fm10k: add Vector RX function
> > >
> > > From: "Chen Jing D(Mark)" <jing.d.chen@intel.com>
> > >
> > > Add func fm10k_recv_raw_pkts_vec to parse raw packets, in which
> > > includes possible chained packets.
> > > Add func fm10k_recv_pkts_vec to receive single mbuf packet.
> > >
> > > Signed-off-by: Chen Jing D(Mark) <jing.d.chen@intel.com>
> > > ---
> > > drivers/net/fm10k/fm10k.h | 1 +
> > > drivers/net/fm10k/fm10k_rxtx_vec.c | 213
> > > ++++++++++++++++++++++++++++++++++++
> > > 2 files changed, 214 insertions(+), 0 deletions(-)
> > >
> > > diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
> > > index d924cae..285254f 100644
> > > --- a/drivers/net/fm10k/fm10k.h
> > > +++ b/drivers/net/fm10k/fm10k.h
> > > @@ -327,4 +327,5 @@ uint16_t fm10k_xmit_pkts(void *tx_queue, struct
> rte_mbuf **tx_pkts,
> > > uint16_t nb_pkts);
> > >
> > > int fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq);
> > > +uint16_t fm10k_recv_pkts_vec(void *, struct rte_mbuf **, uint16_t);
> > > #endif
> > > diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c
> > > b/drivers/net/fm10k/fm10k_rxtx_vec.c
> > > index 581a309..63b34b5 100644
> > > --- a/drivers/net/fm10k/fm10k_rxtx_vec.c
> > > +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
> > > @@ -281,3 +281,216 @@ fm10k_rxq_rearm(struct fm10k_rx_queue *rxq)
> > > /* Update the tail pointer on the NIC */
> > > FM10K_PCI_REG_WRITE(rxq->tail_ptr, rx_id); }
> > > +
> > > +/*
> > > + * vPMD receive routine, now only accept (nb_pkts ==
> > > +RTE_IXGBE_VPMD_RX_BURST)
> > > + * in one loop
> > > + *
> > > + * Notice:
> > > + * - nb_pkts < RTE_IXGBE_VPMD_RX_BURST, just return no packet
>
> Why this limitation? I believe this limitation has already been removed for
> ixgbe, so the same solution should be applicable here
>
> /Bruce
Thanks! I'll change it accordingly.
@@ -327,4 +327,5 @@ uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
int fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq);
+uint16_t fm10k_recv_pkts_vec(void *, struct rte_mbuf **, uint16_t);
#endif
@@ -281,3 +281,216 @@ fm10k_rxq_rearm(struct fm10k_rx_queue *rxq)
/* Update the tail pointer on the NIC */
FM10K_PCI_REG_WRITE(rxq->tail_ptr, rx_id);
}
+
+/*
+ * vPMD receive routine, now only accept (nb_pkts == RTE_IXGBE_VPMD_RX_BURST)
+ * in one loop
+ *
+ * Notice:
+ * - nb_pkts < RTE_IXGBE_VPMD_RX_BURST, just return no packet
+ * - nb_pkts > RTE_IXGBE_VPMD_RX_BURST, only scan RTE_IXGBE_VPMD_RX_BURST
+ * numbers of DD bit
+ * - don't support ol_flags for rss and csum err
+ */
+static inline uint16_t
+fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, uint8_t *split_packet)
+{
+ volatile union fm10k_rx_desc *rxdp;
+ struct rte_mbuf **mbufp;
+ uint16_t nb_pkts_recd;
+ int pos;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint64_t var;
+ __m128i shuf_msk;
+ __m128i dd_check, eop_check;
+ uint16_t next_dd;
+
+ next_dd = rxq->next_dd;
+
+ if (unlikely(nb_pkts < RTE_FM10K_MAX_RX_BURST))
+ return 0;
+
+ /* Just the act of getting into the function from the application is
+ * going to cost about 7 cycles
+ */
+ rxdp = rxq->hw_ring + next_dd;
+
+ _mm_prefetch((const void *)rxdp, _MM_HINT_T0);
+
+ /* See if we need to rearm the RX queue - gives the prefetch a bit
+ * of time to act
+ */
+ if (rxq->rxrearm_nb > RTE_FM10K_RXQ_REARM_THRESH)
+ fm10k_rxq_rearm(rxq);
+
+ /* Before we start moving massive data around, check to see if
+ * there is actually a packet available
+ */
+ if (!(rxdp->d.staterr & FM10K_RXD_STATUS_DD))
+ return 0;
+
+ /* 4 packets DD mask */
+ dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL);
+
+ /* 4 packets EOP mask */
+ eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL);
+
+ /* mask to shuffle from desc. to mbuf */
+ shuf_msk = _mm_set_epi8(
+ 7, 6, 5, 4, /* octet 4~7, 32bits rss */
+ 15, 14, /* octet 14~15, low 16 bits vlan_macip */
+ 13, 12, /* octet 12~13, 16 bits data_len */
+ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
+ 13, 12, /* octet 12~13, low 16 bits pkt_len */
+ 0xFF, 0xFF, /* skip high 16 bits pkt_type */
+ 0xFF, 0xFF /* Skip pkt_type field in shuffle operation */
+ );
+
+ /* Cache is empty -> need to scan the buffer rings, but first move
+ * the next 'n' mbufs into the cache
+ */
+ mbufp = &rxq->sw_ring[next_dd];
+
+ /* A. load 4 packet in one loop
+ * [A*. mask out 4 unused dirty field in desc]
+ * B. copy 4 mbuf point from swring to rx_pkts
+ * C. calc the number of DD bits among the 4 packets
+ * [C*. extract the end-of-packet bit, if requested]
+ * D. fill info. from desc to mbuf
+ */
+ for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
+ pos += RTE_FM10K_DESCS_PER_LOOP,
+ rxdp += RTE_FM10K_DESCS_PER_LOOP) {
+ __m128i descs0[RTE_FM10K_DESCS_PER_LOOP];
+ __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+ __m128i zero, staterr, sterr_tmp1, sterr_tmp2;
+ __m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */
+
+ if (split_packet) {
+ rte_prefetch0(&rx_pkts[pos]->cacheline1);
+ rte_prefetch0(&rx_pkts[pos + 1]->cacheline1);
+ rte_prefetch0(&rx_pkts[pos + 2]->cacheline1);
+ rte_prefetch0(&rx_pkts[pos + 3]->cacheline1);
+ }
+
+ /* B.1 load 1 mbuf point */
+ mbp1 = _mm_loadu_si128((__m128i *)&mbufp[pos]);
+
+ /* Read desc statuses backwards to avoid race condition */
+ /* A.1 load 4 pkts desc */
+ descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
+
+ /* B.2 copy 2 mbuf point into rx_pkts */
+ _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
+
+ /* B.1 load 1 mbuf point */
+ mbp2 = _mm_loadu_si128((__m128i *)&mbufp[pos+2]);
+
+ descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
+ /* B.1 load 2 mbuf point */
+ descs0[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
+ descs0[0] = _mm_loadu_si128((__m128i *)(rxdp));
+
+ /* B.2 copy 2 mbuf point into rx_pkts */
+ _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2);
+
+ /* avoid compiler reorder optimization */
+ rte_compiler_barrier();
+
+ /* D.1 pkt 3,4 convert format from desc to pktmbuf */
+ pkt_mb4 = _mm_shuffle_epi8(descs0[3], shuf_msk);
+ pkt_mb3 = _mm_shuffle_epi8(descs0[2], shuf_msk);
+
+ /* C.1 4=>2 filter staterr info only */
+ sterr_tmp2 = _mm_unpackhi_epi32(descs0[3], descs0[2]);
+ /* C.1 4=>2 filter staterr info only */
+ sterr_tmp1 = _mm_unpackhi_epi32(descs0[1], descs0[0]);
+
+ /* set ol_flags with vlan packet type */
+ fm10k_desc_to_olflags_v(descs0, &rx_pkts[pos]);
+
+ /* D.1 pkt 1,2 convert format from desc to pktmbuf */
+ pkt_mb2 = _mm_shuffle_epi8(descs0[1], shuf_msk);
+ pkt_mb1 = _mm_shuffle_epi8(descs0[0], shuf_msk);
+
+ /* C.2 get 4 pkts staterr value */
+ zero = _mm_xor_si128(dd_check, dd_check);
+ staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
+
+ /* D.3 copy final 3,4 data to rx_pkts */
+ _mm_storeu_si128((void *)&rx_pkts[pos+3]->rx_descriptor_fields1,
+ pkt_mb4);
+ _mm_storeu_si128((void *)&rx_pkts[pos+2]->rx_descriptor_fields1,
+ pkt_mb3);
+
+ /* C* extract and record EOP bit */
+ if (split_packet) {
+ __m128i eop_shuf_mask = _mm_set_epi8(
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0x04, 0x0C, 0x00, 0x08
+ );
+
+ /* and with mask to extract bits, flipping 1-0 */
+ __m128i eop_bits = _mm_andnot_si128(staterr, eop_check);
+ /* the staterr values are not in order, as the count
+ * count of dd bits doesn't care. However, for end of
+ * packet tracking, we do care, so shuffle. This also
+ * compresses the 32-bit values to 8-bit
+ */
+ eop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask);
+ /* store the resulting 32-bit value */
+ *(int *)split_packet = _mm_cvtsi128_si32(eop_bits);
+ split_packet += RTE_FM10K_DESCS_PER_LOOP;
+
+ /* zero-out next pointers */
+ rx_pkts[pos]->next = NULL;
+ rx_pkts[pos + 1]->next = NULL;
+ rx_pkts[pos + 2]->next = NULL;
+ rx_pkts[pos + 3]->next = NULL;
+ }
+
+ /* C.3 calc available number of desc */
+ staterr = _mm_and_si128(staterr, dd_check);
+ staterr = _mm_packs_epi32(staterr, zero);
+
+ /* D.3 copy final 1,2 data to rx_pkts */
+ _mm_storeu_si128((void *)&rx_pkts[pos+1]->rx_descriptor_fields1,
+ pkt_mb2);
+ _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
+ pkt_mb1);
+
+ fm10k_desc_to_pktype_v(descs0, &rx_pkts[pos]);
+
+ /* C.4 calc avaialbe number of desc */
+ var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
+ nb_pkts_recd += var;
+ if (likely(var != RTE_FM10K_DESCS_PER_LOOP))
+ break;
+ }
+
+ /* Update our internal tail pointer */
+ rxq->next_dd = (uint16_t)(rxq->next_dd + nb_pkts_recd);
+ rxq->next_dd = (uint16_t)(rxq->next_dd & (rxq->nb_desc - 1));
+ rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
+
+ return nb_pkts_recd;
+}
+
+/* vPMD receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP)
+ *
+ * Notice:
+ * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
+ * numbers of DD bit
+ * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
+ * - don't support ol_flags for rss and csum err
+ */
+uint16_t
+fm10k_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ return fm10k_recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
+}