From: Phil Yang <phil.yang@arm.com>
PMD Rx queue descriptor contains two mlx5_mprq_buf fields, which
are the multi-packet RQ buffer header pointers. It uses the common
rte_atomic_XXX functions to make sure the refcnt access is atomic.
The common rte_atomic_XXX functions are full barriers on aarch64.
Optimized it with one-way barrier to improve performance.
Fixes: 7d6bf6b866b8 ("net/mlx5: add Multi-Packet Rx support")
Cc: stable@dpdk.org
Suggested-by: Gavin Hu <gavin.hu@arm.com>
Signed-off-by: Phil Yang <phil.yang@arm.com>
Reviewed-by: Gavin Hu <gavin.hu@arm.com>
---
drivers/net/mlx5/mlx5_rxq.c | 2 +-
drivers/net/mlx5/mlx5_rxtx.c | 16 +++++++++-------
drivers/net/mlx5/mlx5_rxtx.h | 2 +-
3 files changed, 11 insertions(+), 9 deletions(-)
@@ -1539,7 +1539,7 @@ mlx5_mprq_buf_init(struct rte_mempool *mp, void *opaque_arg,
memset(_m, 0, sizeof(*buf));
buf->mp = mp;
- rte_atomic16_set(&buf->refcnt, 1);
+ __atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED);
for (j = 0; j != strd_n; ++j) {
shinfo = &buf->shinfos[j];
shinfo->free_cb = mlx5_mprq_buf_free_cb;
@@ -1592,10 +1592,11 @@ mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque)
{
struct mlx5_mprq_buf *buf = opaque;
- if (rte_atomic16_read(&buf->refcnt) == 1) {
+ if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) == 1) {
rte_mempool_put(buf->mp, buf);
- } else if (rte_atomic16_add_return(&buf->refcnt, -1) == 0) {
- rte_atomic16_set(&buf->refcnt, 1);
+ } else if (unlikely(__atomic_sub_fetch(&buf->refcnt, 1,
+ __ATOMIC_RELAXED) == 0)) {
+ __atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED);
rte_mempool_put(buf->mp, buf);
}
}
@@ -1676,7 +1677,8 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
if (consumed_strd == strd_n) {
/* Replace WQE only if the buffer is still in use. */
- if (rte_atomic16_read(&buf->refcnt) > 1) {
+ if (__atomic_load_n(&buf->refcnt,
+ __ATOMIC_RELAXED) > 1) {
mprq_buf_replace(rxq, rq_ci & wq_mask, strd_n);
/* Release the old buffer. */
mlx5_mprq_buf_free(buf);
@@ -1766,9 +1768,9 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
void *buf_addr;
/* Increment the refcnt of the whole chunk. */
- rte_atomic16_add_return(&buf->refcnt, 1);
- MLX5_ASSERT((uint16_t)rte_atomic16_read(&buf->refcnt) <=
- strd_n + 1);
+ __atomic_add_fetch(&buf->refcnt, 1, __ATOMIC_ACQUIRE);
+ MLX5_ASSERT(__atomic_load_n(&buf->refcnt,
+ __ATOMIC_RELAXED) <= strd_n + 1);
buf_addr = RTE_PTR_SUB(addr, headroom_sz);
/*
* MLX5 device doesn't use iova but it is necessary in a
@@ -78,7 +78,7 @@ struct rxq_zip {
/* Multi-Packet RQ buffer header. */
struct mlx5_mprq_buf {
struct rte_mempool *mp;
- rte_atomic16_t refcnt; /* Atomically accessed refcnt. */
+ uint16_t refcnt; /* Atomically accessed refcnt. */
uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */
struct rte_mbuf_ext_shared_info shinfos[];
/*