@@ -390,10 +390,10 @@ mlx5_flow_counters_mng_close(struct mlx5_ibv_shared *sh)
(mlx5_devx_cmd_destroy(pool->min_dcs));
}
for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) {
- if (pool->counters_raw[j].action)
+ if (MLX5_POOL_GET_CNT(pool, j)->action)
claim_zero
(mlx5_glue->destroy_flow_action
- (pool->counters_raw[j].action));
+ (MLX5_POOL_GET_CNT(pool, j)->action));
if (!batch && MLX5_GET_POOL_CNT_EXT
(pool, j)->dcs)
claim_zero(mlx5_devx_cmd_destroy
@@ -240,6 +240,18 @@ struct mlx5_drop {
#define MLX5_COUNTERS_PER_POOL 512
#define MLX5_MAX_PENDING_QUERIES 4
#define MLX5_CNT_CONTAINER_RESIZE 64
+#define CNT_SIZE (sizeof(struct mlx5_flow_counter))
+#define CNTEXT_SIZE (sizeof(struct mlx5_flow_counter_ext))
+
+#define CNT_POOL_TYPE_EXT (1 << 0)
+#define IS_EXT_POOL(pool) (((pool)->type) & CNT_POOL_TYPE_EXT)
+#define MLX5_CNT_LEN(pool) \
+ (CNT_SIZE + (IS_EXT_POOL((pool)) ? CNTEXT_SIZE : 0))
+#define MLX5_POOL_GET_CNT(pool, index) \
+ ((struct mlx5_flow_counter *) \
+ ((char *)((pool) + 1) + (index) * (MLX5_CNT_LEN(pool))))
+#define MLX5_CNT_ARRAY_IDX(pool, cnt) \
+ ((int)(((char *)(cnt) - (char *)((pool) + 1)) / MLX5_CNT_LEN((pool)))) \
/*
* The pool index and offset of counter in the pool array makes up the
* counter index. In case the counter is from pool 0 and offset 0, it
@@ -248,11 +260,10 @@ struct mlx5_drop {
*/
#define MLX5_MAKE_CNT_IDX(pi, offset) \
((pi) * MLX5_COUNTERS_PER_POOL + (offset) + 1)
-#define MLX5_CNT_TO_CNT_EXT(pool, cnt) (&((struct mlx5_flow_counter_ext *) \
- ((pool) + 1))[((cnt) - (pool)->counters_raw)])
+#define MLX5_CNT_TO_CNT_EXT(cnt) \
+ ((struct mlx5_flow_counter_ext *)((cnt) + 1))
#define MLX5_GET_POOL_CNT_EXT(pool, offset) \
- (&((struct mlx5_flow_counter_ext *) \
- ((pool) + 1))[offset])
+ MLX5_CNT_TO_CNT_EXT(MLX5_POOL_GET_CNT((pool), (offset)))
struct mlx5_flow_counter_pool;
@@ -305,10 +316,10 @@ struct mlx5_flow_counter_pool {
rte_atomic64_t start_query_gen; /* Query start round. */
rte_atomic64_t end_query_gen; /* Query end round. */
uint32_t index; /* Pool index in container. */
+ uint32_t type: 2;
rte_spinlock_t sl; /* The pool lock. */
struct mlx5_counter_stats_raw *raw;
struct mlx5_counter_stats_raw *raw_hw; /* The raw on HW working. */
- struct mlx5_flow_counter counters_raw[MLX5_COUNTERS_PER_POOL];
/* The pool counters memory. */
};
@@ -3854,7 +3854,7 @@ flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
MLX5_ASSERT(pool);
if (ppool)
*ppool = pool;
- return &pool->counters_raw[idx % MLX5_COUNTERS_PER_POOL];
+ return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
}
/**
@@ -4062,7 +4062,7 @@ _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
MLX5_ASSERT(pool);
if (counter < MLX5_CNT_BATCH_OFFSET) {
- cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
+ cnt_ext = MLX5_CNT_TO_CNT_EXT(cnt);
if (priv->counter_fallback)
return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
0, pkts, bytes, 0, NULL, NULL, 0);
@@ -4078,7 +4078,7 @@ _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
*pkts = 0;
*bytes = 0;
} else {
- offset = cnt - &pool->counters_raw[0];
+ offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
*pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
*bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
}
@@ -4118,9 +4118,9 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
return NULL;
}
size = sizeof(*pool);
+ size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
if (!batch)
- size += MLX5_COUNTERS_PER_POOL *
- sizeof(struct mlx5_flow_counter_ext);
+ size += MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE;
pool = rte_calloc(__func__, 1, size, 0);
if (!pool) {
rte_errno = ENOMEM;
@@ -4131,6 +4131,9 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
pool->raw = cont->init_mem_mng->raws + n_valid %
MLX5_CNT_CONTAINER_RESIZE;
pool->raw_hw = NULL;
+ pool->type = 0;
+ if (!batch)
+ pool->type |= CNT_POOL_TYPE_EXT;
rte_spinlock_init(&pool->sl);
/*
* The generation of the new allocated counters in this pool is 0, 2 in
@@ -4202,7 +4205,7 @@ flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
(int64_t)(uintptr_t)dcs);
}
i = dcs->id % MLX5_COUNTERS_PER_POOL;
- cnt = &pool->counters_raw[i];
+ cnt = MLX5_POOL_GET_CNT(pool, i);
TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
*cnt_free = cnt;
@@ -4222,10 +4225,10 @@ flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
}
pool = TAILQ_FIRST(&cont->pool_list);
for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
- cnt = &pool->counters_raw[i];
+ cnt = MLX5_POOL_GET_CNT(pool, i);
TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
}
- *cnt_free = &pool->counters_raw[0];
+ *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
return cont;
}
@@ -4343,14 +4346,14 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
pool = TAILQ_FIRST(&cont->pool_list);
}
if (!batch)
- cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
+ cnt_ext = MLX5_CNT_TO_CNT_EXT(cnt_free);
/* Create a DV counter action only in the first time usage. */
if (!cnt_free->action) {
uint16_t offset;
struct mlx5_devx_obj *dcs;
if (batch) {
- offset = cnt_free - &pool->counters_raw[0];
+ offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
dcs = pool->min_dcs;
} else {
offset = 0;
@@ -4364,7 +4367,7 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
}
}
cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
- (cnt_free - pool->counters_raw));
+ MLX5_CNT_ARRAY_IDX(pool, cnt_free));
cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
/* Update the counter reset values. */
if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
@@ -4407,7 +4410,7 @@ flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
MLX5_ASSERT(pool);
if (counter < MLX5_CNT_BATCH_OFFSET) {
- cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
+ cnt_ext = MLX5_CNT_TO_CNT_EXT(cnt);
if (cnt_ext && --cnt_ext->ref_cnt)
return;
}
@@ -64,7 +64,7 @@ flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
MLX5_ASSERT(pool);
if (ppool)
*ppool = pool;
- return &pool->counters_raw[idx % MLX5_COUNTERS_PER_POOL];
+ return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
}
/**
@@ -207,16 +207,16 @@ flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
if (!pool)
return 0;
for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
- cnt = &pool->counters_raw[i];
+ cnt = MLX5_POOL_GET_CNT(pool, i);
TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
}
- cnt = &pool->counters_raw[0];
+ cnt = MLX5_POOL_GET_CNT(pool, 0);
cont->pools[n_valid] = pool;
pool_idx = n_valid;
rte_atomic16_add(&cont->n_valid, 1);
TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
}
- i = cnt - pool->counters_raw;
+ i = MLX5_CNT_ARRAY_IDX(pool, cnt);
cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
cnt_ext->id = id;
cnt_ext->shared = shared;
@@ -251,7 +251,7 @@ flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
cnt = flow_verbs_counter_get_by_idx(dev, counter,
&pool);
- cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
+ cnt_ext = MLX5_CNT_TO_CNT_EXT(cnt);
if (--cnt_ext->ref_cnt == 0) {
#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs));
@@ -282,7 +282,7 @@ flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
(dev, flow->counter, &pool);
struct mlx5_flow_counter_ext *cnt_ext = MLX5_CNT_TO_CNT_EXT
- (pool, cnt);
+ (cnt);
struct rte_flow_query_count *qc = data;
uint64_t counters[2] = {0, 0};
#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
@@ -1090,12 +1090,12 @@ flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
}
#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
- cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
+ cnt_ext = MLX5_CNT_TO_CNT_EXT(cnt);
counter.counter_set_handle = cnt_ext->cs->handle;
flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
- cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
+ cnt_ext = MLX5_CNT_TO_CNT_EXT(cnt);
counter.counters = cnt_ext->cs;
flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
#endif