@@ -783,7 +783,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
{
const uint16_t q_n = 1 << rxq->cqe_n;
const uint16_t q_mask = q_n - 1;
- unsigned int pos;
+ unsigned int pos, adj;
uint64_t n = 0;
uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
uint16_t nocmp_n = 0;
@@ -866,7 +866,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
__vector unsigned char pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
__vector unsigned char op_own, op_own_tmp1, op_own_tmp2;
__vector unsigned char opcode, owner_mask, invalid_mask;
- __vector unsigned char comp_mask;
+ __vector unsigned char comp_mask, mini_mask;
__vector unsigned char mask;
#ifdef MLX5_PMD_SOFT_COUNTERS
const __vector unsigned char lower_half = {
@@ -1174,6 +1174,16 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
(__vector unsigned long)mask);
/* D.3 check error in opcode. */
+ adj = (comp_idx != MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n);
+ mask = (__vector unsigned char)(__vector unsigned long){
+ (adj * sizeof(uint16_t) * 8), 0};
+ lshift = vec_splat((__vector unsigned long)mask, 0);
+ shmask = vec_cmpgt(shmax, lshift);
+ mini_mask = (__vector unsigned char)
+ vec_sl((__vector unsigned long)invalid_mask, lshift);
+ mini_mask = (__vector unsigned char)
+ vec_sel((__vector unsigned long)shmask,
+ (__vector unsigned long)mini_mask, shmask);
opcode = (__vector unsigned char)
vec_cmpeq((__vector unsigned int)resp_err_check,
(__vector unsigned int)opcode);
@@ -1182,7 +1192,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
(__vector unsigned int)zero);
opcode = (__vector unsigned char)
vec_andc((__vector unsigned long)opcode,
- (__vector unsigned long)invalid_mask);
+ (__vector unsigned long)mini_mask);
/* D.4 mark if any error is set */
*err |= ((__vector unsigned long)opcode)[0];
@@ -524,7 +524,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
{
const uint16_t q_n = 1 << rxq->cqe_n;
const uint16_t q_mask = q_n - 1;
- unsigned int pos;
+ unsigned int pos, adj;
uint64_t n = 0;
uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
uint16_t nocmp_n = 0;
@@ -616,7 +616,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
pos += MLX5_VPMD_DESCS_PER_LOOP) {
uint16x4_t op_own;
uint16x4_t opcode, owner_mask, invalid_mask;
- uint16x4_t comp_mask;
+ uint16x4_t comp_mask, mini_mask;
uint16x4_t mask;
uint16x4_t byte_cnt;
uint32x4_t ptype_info, flow_tag;
@@ -780,8 +780,12 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
-1UL >> (n * sizeof(uint16_t) * 8) : 0);
invalid_mask = vorr_u16(invalid_mask, mask);
/* D.3 check error in opcode. */
+ adj = (comp_idx != MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n);
+ mask = vcreate_u16(adj ?
+ -1UL >> ((n + 1) * sizeof(uint16_t) * 8) : -1UL);
+ mini_mask = vand_u16(invalid_mask, mask);
opcode = vceq_u16(resp_err_check, opcode);
- opcode = vbic_u16(opcode, invalid_mask);
+ opcode = vbic_u16(opcode, mini_mask);
/* D.4 mark if any error is set */
*err |= vget_lane_u64(vreinterpret_u64_u16(opcode), 0);
/* C.4 fill in mbuf - rearm_data and packet_type. */
@@ -523,7 +523,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
{
const uint16_t q_n = 1 << rxq->cqe_n;
const uint16_t q_mask = q_n - 1;
- unsigned int pos;
+ unsigned int pos, adj;
uint64_t n = 0;
uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
uint16_t nocmp_n = 0;
@@ -591,7 +591,7 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
__m128i op_own, op_own_tmp1, op_own_tmp2;
__m128i opcode, owner_mask, invalid_mask;
- __m128i comp_mask;
+ __m128i comp_mask, mini_mask;
__m128i mask;
#ifdef MLX5_PMD_SOFT_COUNTERS
__m128i byte_cnt;
@@ -729,9 +729,12 @@ rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
mask = _mm_sll_epi64(ones, mask);
invalid_mask = _mm_or_si128(invalid_mask, mask);
/* D.3 check error in opcode. */
+ adj = (comp_idx != MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n);
+ mask = _mm_set_epi64x(0, adj * sizeof(uint16_t) * 8);
+ mini_mask = _mm_sll_epi64(invalid_mask, mask);
opcode = _mm_cmpeq_epi32(resp_err_check, opcode);
opcode = _mm_packs_epi32(opcode, zero);
- opcode = _mm_andnot_si128(invalid_mask, opcode);
+ opcode = _mm_andnot_si128(mini_mask, opcode);
/* D.4 mark if any error is set */
*err |= _mm_cvtsi128_si64(opcode);
/* D.5 fill in mbuf - rearm_data and packet_type. */