@@ -512,6 +512,32 @@
}
/**
+ * DV flow counter mode detect and config.
+ *
+ * @param dev
+ * Pointer to rte_eth_dev structure.
+ *
+ */
+static void
+mlx5_flow_counter_mode_config(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ /* If devx is not supported, counters are not working. */
+ if (!priv->config.devx)
+ return;
+ priv->counter_fallback = 0;
+ if (!priv->config.hca_attr.flow_counters_dump ||
+ (mlx5_flow_discover_counter_offset_support(dev) == -ENOTSUP))
+ priv->counter_fallback = 1;
+#ifndef HAVE_IBV_DEVX_ASYNC
+ priv->counter_fallback = 1;
+#endif
+ if (priv->counter_fallback)
+ DRV_LOG(INFO, "Use fall-back DV counter management");
+}
+
+/**
* Spawn an Ethernet device from Verbs information.
*
* @param dpdk_dev
@@ -979,19 +1005,11 @@
DRV_LOG(INFO, "Rx CQE padding is enabled");
}
if (config->devx) {
- priv->counter_fallback = 0;
err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config->hca_attr);
if (err) {
err = -err;
goto error;
}
- if (!config->hca_attr.flow_counters_dump)
- priv->counter_fallback = 1;
-#ifndef HAVE_IBV_DEVX_ASYNC
- priv->counter_fallback = 1;
-#endif
- if (priv->counter_fallback)
- DRV_LOG(INFO, "Use fall-back DV counter management");
/* Check for LRO support. */
if (config->dest_tir && config->hca_attr.lro_cap &&
config->dv_flow_en) {
@@ -1364,6 +1382,8 @@
goto error;
}
}
+ if (priv->config.dv_flow_en)
+ mlx5_flow_counter_mode_config(eth_dev);
return eth_dev;
error:
if (priv) {
@@ -458,20 +458,18 @@ struct mlx5_flow_id_pool *
static void
mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh)
{
- int i, j;
+ int i;
memset(&sh->cmng, 0, sizeof(sh->cmng));
TAILQ_INIT(&sh->cmng.flow_counters);
- for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i) {
- sh->cmng.ccont[i].min_id = MLX5_CNT_BATCH_OFFSET;
- sh->cmng.ccont[i].max_id = -1;
- sh->cmng.ccont[i].last_pool_idx = POOL_IDX_INVALID;
- TAILQ_INIT(&sh->cmng.ccont[i].pool_list);
- rte_spinlock_init(&sh->cmng.ccont[i].resize_sl);
- for (j = 0; j < MLX5_COUNTER_TYPE_MAX; j++)
- TAILQ_INIT(&sh->cmng.ccont[i].counters[j]);
- rte_spinlock_init(&sh->cmng.ccont[i].csl);
- }
+ sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET;
+ sh->cmng.max_id = -1;
+ sh->cmng.last_pool_idx = POOL_IDX_INVALID;
+ TAILQ_INIT(&sh->cmng.pool_list);
+ rte_spinlock_init(&sh->cmng.resize_sl);
+ for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++)
+ TAILQ_INIT(&sh->cmng.counters[i]);
+ rte_spinlock_init(&sh->cmng.csl);
}
/**
@@ -501,7 +499,6 @@ struct mlx5_flow_id_pool *
mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh)
{
struct mlx5_counter_stats_mem_mng *mng;
- int i;
int j;
int retries = 1024;
@@ -512,15 +509,13 @@ struct mlx5_flow_id_pool *
break;
rte_pause();
}
- for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i) {
+
+ if (sh->cmng.pools) {
struct mlx5_flow_counter_pool *pool;
- uint32_t batch = (i == MLX5_CCONT_TYPE_BATCH);
- if (!sh->cmng.ccont[i].pools)
- continue;
- pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
+ pool = TAILQ_FIRST(&sh->cmng.pool_list);
while (pool) {
- if (batch && pool->min_dcs)
+ if (!IS_EXT_POOL(pool) && pool->min_dcs)
claim_zero(mlx5_devx_cmd_destroy
(pool->min_dcs));
for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) {
@@ -529,17 +524,17 @@ struct mlx5_flow_id_pool *
(mlx5_glue->destroy_flow_action
(MLX5_POOL_GET_CNT
(pool, j)->action));
- if (!batch && MLX5_GET_POOL_CNT_EXT
+ if (IS_EXT_POOL(pool) && MLX5_GET_POOL_CNT_EXT
(pool, j)->dcs)
claim_zero(mlx5_devx_cmd_destroy
(MLX5_GET_POOL_CNT_EXT
(pool, j)->dcs));
}
- TAILQ_REMOVE(&sh->cmng.ccont[i].pool_list, pool, next);
+ TAILQ_REMOVE(&sh->cmng.pool_list, pool, next);
mlx5_free(pool);
- pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
+ pool = TAILQ_FIRST(&sh->cmng.pool_list);
}
- mlx5_free(sh->cmng.ccont[i].pools);
+ mlx5_free(sh->cmng.pools);
}
mng = LIST_FIRST(&sh->cmng.mem_mngs);
while (mng) {
@@ -280,8 +280,10 @@ struct mlx5_drop {
#define MLX5_AGING_TIME_DELAY 7
#define CNT_POOL_TYPE_EXT (1 << 0)
#define CNT_POOL_TYPE_AGE (1 << 1)
+
#define IS_EXT_POOL(pool) (((pool)->type) & CNT_POOL_TYPE_EXT)
#define IS_AGE_POOL(pool) (((pool)->type) & CNT_POOL_TYPE_AGE)
+
#define MLX5_CNT_LEN(pool) \
(CNT_SIZE + \
(IS_AGE_POOL(pool) ? AGE_SIZE : 0) + \
@@ -324,14 +326,6 @@ enum {
AGE_TMOUT, /* Timeout, wait for rte_flow_get_aged_flows and destroy. */
};
-#define MLX5_CNT_CONTAINER(sh, batch) (&(sh)->cmng.ccont[batch])
-
-enum {
- MLX5_CCONT_TYPE_SINGLE,
- MLX5_CCONT_TYPE_BATCH,
- MLX5_CCONT_TYPE_MAX,
-};
-
enum mlx5_counter_type {
MLX5_COUNTER_TYPE_ORIGIN,
MLX5_COUNTER_TYPE_AGE,
@@ -377,7 +371,6 @@ struct mlx5_flow_counter {
/* Extend counters information for none batch fallback counters. */
struct mlx5_flow_counter_ext {
- uint32_t skipped:1; /* This counter is skipped or not. */
#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
struct ibv_counter_set *cs;
#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
@@ -397,9 +390,8 @@ struct mlx5_flow_counter_pool {
rte_atomic64_t a64_dcs;
};
/* The devx object of the minimum counter ID. */
- uint32_t index:28; /* Pool index in container. */
+ uint32_t index:29; /* Pool index in container. */
uint32_t type:2; /* Memory type behind the counter array. */
- uint32_t skip_cnt:1; /* Pool contains skipped counter. */
volatile uint32_t query_gen:1; /* Query round. */
rte_spinlock_t sl; /* The pool lock. */
struct mlx5_counter_stats_raw *raw;
@@ -419,15 +411,14 @@ struct mlx5_counter_stats_mem_mng {
/* Raw memory structure for the counter statistics values of a pool. */
struct mlx5_counter_stats_raw {
LIST_ENTRY(mlx5_counter_stats_raw) next;
- int min_dcs_id;
struct mlx5_counter_stats_mem_mng *mem_mng;
volatile struct flow_counter_stats *data;
};
TAILQ_HEAD(mlx5_counter_pools, mlx5_flow_counter_pool);
-/* Container structure for counter pools. */
-struct mlx5_pools_container {
+/* Counter global management structure. */
+struct mlx5_flow_counter_mng {
rte_atomic16_t n_valid; /* Number of valid pools. */
uint16_t n; /* Number of pools. */
uint16_t last_pool_idx; /* Last used pool index */
@@ -441,14 +432,8 @@ struct mlx5_pools_container {
struct mlx5_flow_counter_pool **pools; /* Counter pool array. */
struct mlx5_counter_stats_mem_mng *mem_mng;
/* Hold the memory management for the next allocated pools raws. */
-};
-
-/* Counter global management structure. */
-struct mlx5_flow_counter_mng {
- struct mlx5_pools_container ccont[MLX5_CCONT_TYPE_MAX];
struct mlx5_counters flow_counters; /* Legacy flow counter list. */
uint8_t pending_queries;
- uint8_t batch;
uint16_t pool_index;
uint8_t query_thread_on;
LIST_HEAD(mem_mngs, mlx5_counter_stats_mem_mng) mem_mngs;
@@ -838,6 +823,8 @@ struct mlx5_priv {
struct mlx5_flow_meters flow_meters; /* MTR list. */
uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */
uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */
+ void *cnt_action; /* Counter action to validate invalid offset. */
+ struct mlx5_devx_obj *cnt_dcs; /* Counter validate devx object. */
struct mlx5_mp_id mp_id; /* ID of a multi-process process */
LIST_HEAD(fdir, mlx5_fdir_flow) fdir_flows; /* fdir flows. */
};
@@ -5883,26 +5883,6 @@ struct mlx5_meter_domains_infos *
#define MLX5_POOL_QUERY_FREQ_US 1000000
/**
- * Get number of all validate pools.
- *
- * @param[in] sh
- * Pointer to mlx5_dev_ctx_shared object.
- *
- * @return
- * The number of all validate pools.
- */
-static uint32_t
-mlx5_get_all_valid_pool_count(struct mlx5_dev_ctx_shared *sh)
-{
- int i;
- uint32_t pools_n = 0;
-
- for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i)
- pools_n += rte_atomic16_read(&sh->cmng.ccont[i].n_valid);
- return pools_n;
-}
-
-/**
* Set the periodic procedure for triggering asynchronous batch queries for all
* the counter pools.
*
@@ -5914,7 +5894,7 @@ struct mlx5_meter_domains_infos *
{
uint32_t pools_n, us;
- pools_n = mlx5_get_all_valid_pool_count(sh);
+ pools_n = rte_atomic16_read(&sh->cmng.n_valid);
us = MLX5_POOL_QUERY_FREQ_US / pools_n;
DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
@@ -5936,31 +5916,20 @@ struct mlx5_meter_domains_infos *
mlx5_flow_query_alarm(void *arg)
{
struct mlx5_dev_ctx_shared *sh = arg;
- struct mlx5_devx_obj *dcs;
- uint16_t offset;
int ret;
- uint8_t batch = sh->cmng.batch;
uint16_t pool_index = sh->cmng.pool_index;
- struct mlx5_pools_container *cont;
+ struct mlx5_flow_counter_mng *cmng = &sh->cmng;
struct mlx5_flow_counter_pool *pool;
- int cont_loop = MLX5_CCONT_TYPE_MAX;
if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
goto set_alarm;
-next_container:
- cont = MLX5_CNT_CONTAINER(sh, batch);
- rte_spinlock_lock(&cont->resize_sl);
- if (!cont->pools) {
- rte_spinlock_unlock(&cont->resize_sl);
- /* Check if all the containers are empty. */
- if (unlikely(--cont_loop == 0))
- goto set_alarm;
- batch ^= 0x1;
- pool_index = 0;
- goto next_container;
+ rte_spinlock_lock(&cmng->resize_sl);
+ if (!cmng->pools) {
+ rte_spinlock_unlock(&cmng->resize_sl);
+ goto set_alarm;
}
- pool = cont->pools[pool_index];
- rte_spinlock_unlock(&cont->resize_sl);
+ pool = cmng->pools[pool_index];
+ rte_spinlock_unlock(&cmng->resize_sl);
if (pool->raw_hw)
/* There is a pool query in progress. */
goto set_alarm;
@@ -5969,14 +5938,6 @@ struct mlx5_meter_domains_infos *
if (!pool->raw_hw)
/* No free counter statistics raw memory. */
goto set_alarm;
- dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read
- (&pool->a64_dcs);
- if (dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) {
- /* Pool without valid counter. */
- pool->raw_hw = NULL;
- goto next_pool;
- }
- offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL;
/*
* Identify the counters released between query trigger and query
* handle more effiecntly. The counter released in this gap period
@@ -5984,11 +5945,12 @@ struct mlx5_meter_domains_infos *
* will not be taken into account.
*/
pool->query_gen++;
- ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL -
- offset, NULL, NULL,
+ ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
+ MLX5_COUNTERS_PER_POOL,
+ NULL, NULL,
pool->raw_hw->mem_mng->dm->id,
(void *)(uintptr_t)
- (pool->raw_hw->data + offset),
+ pool->raw_hw->data,
sh->devx_comp,
(uint64_t)(uintptr_t)pool);
if (ret) {
@@ -5997,17 +5959,12 @@ struct mlx5_meter_domains_infos *
pool->raw_hw = NULL;
goto set_alarm;
}
- pool->raw_hw->min_dcs_id = dcs->id;
LIST_REMOVE(pool->raw_hw, next);
sh->cmng.pending_queries++;
-next_pool:
pool_index++;
- if (pool_index >= rte_atomic16_read(&cont->n_valid)) {
- batch ^= 0x1;
+ if (pool_index >= rte_atomic16_read(&cmng->n_valid))
pool_index = 0;
- }
set_alarm:
- sh->cmng.batch = batch;
sh->cmng.pool_index = pool_index;
mlx5_set_query_alarm(sh);
}
@@ -6095,8 +6052,7 @@ struct mlx5_meter_domains_infos *
(struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
struct mlx5_counter_stats_raw *raw_to_free;
uint8_t query_gen = pool->query_gen ^ 1;
- struct mlx5_pools_container *cont =
- MLX5_CNT_CONTAINER(sh, !IS_EXT_POOL(pool));
+ struct mlx5_flow_counter_mng *cmng = &sh->cmng;
enum mlx5_counter_type cnt_type =
IS_AGE_POOL(pool) ? MLX5_COUNTER_TYPE_AGE :
MLX5_COUNTER_TYPE_ORIGIN;
@@ -6113,10 +6069,10 @@ struct mlx5_meter_domains_infos *
/* Be sure the new raw counters data is updated in memory. */
rte_io_wmb();
if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
- rte_spinlock_lock(&cont->csl);
- TAILQ_CONCAT(&cont->counters[cnt_type],
+ rte_spinlock_lock(&cmng->csl);
+ TAILQ_CONCAT(&cmng->counters[cnt_type],
&pool->counters[query_gen], next);
- rte_spinlock_unlock(&cont->csl);
+ rte_spinlock_unlock(&cmng->csl);
}
}
LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
@@ -6238,6 +6194,80 @@ struct mlx5_meter_domains_infos *
}
/**
+ * Validate if batch counter supported in root table.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_discover_counter_offset_support(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow_attr attr = {
+ .group = 0,
+ .ingress = 1,
+ };
+ struct rte_flow_item items[] = {
+ [0] = {
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ },
+ };
+ struct rte_flow_action actions[] = {
+ [0] = {
+ .type = (enum rte_flow_action_type)
+ MLX5_RTE_FLOW_ACTION_TYPE_COUNT,
+ },
+ [1] = {
+ .type = RTE_FLOW_ACTION_TYPE_JUMP,
+ .conf = &(struct rte_flow_action_jump){
+ .group = 1,
+ },
+ },
+ [2] = {
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ },
+ };
+ int ret = 0;
+ uint32_t flow_idx;
+ struct rte_flow_error error;
+
+ flow_idx = flow_list_create(dev, NULL, &attr, items,
+ actions, true, &error);
+ /*
+ * If batch counter with offset is not supported, the driver will not
+ * validate the invalid offset value, flow create will successfully.
+ * In this case, it means batch counter is not supported in root table.
+ *
+ * Otherwise, if flow create failed with other cases, report error
+ * message.
+ */
+ if (flow_idx) {
+ flow_list_destroy(dev, NULL, flow_idx);
+ DRV_LOG(WARNING, "Batch counter is not supported in root "
+ "table. Switch to fallback mode.");
+ rte_errno = ENOTSUP;
+ ret = -rte_errno;
+ } else {
+ if (errno != EINVAL)
+ DRV_LOG(ERR, "Counter may not work correctly as "
+ "validate fail with unknown reason.");
+ ret = 0;
+ }
+ if (priv->cnt_action) {
+ mlx5_flow_os_destroy_flow_action(priv->cnt_action);
+ priv->cnt_action = NULL;
+ }
+ if (priv->cnt_dcs) {
+ mlx5_devx_cmd_destroy(priv->cnt_dcs);
+ priv->cnt_dcs = NULL;
+ }
+ return ret;
+}
+
+/**
* Dump flow raw hw data to file
*
* @param[in] dev
@@ -35,6 +35,7 @@ enum mlx5_rte_flow_action_type {
MLX5_RTE_FLOW_ACTION_TYPE_MARK,
MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
+ MLX5_RTE_FLOW_ACTION_TYPE_COUNT,
};
/* Matches on selected register. */
@@ -1069,4 +1070,5 @@ int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr);
int mlx5_flow_meter_flush(struct rte_eth_dev *dev,
struct rte_mtr_error *error);
+int mlx5_flow_discover_counter_offset_support(struct rte_eth_dev *dev);
#endif /* RTE_PMD_MLX5_FLOW_H_ */
@@ -4168,19 +4168,13 @@ struct field_modify_info modify_tcp[] = {
struct mlx5_flow_counter_pool **ppool)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_pools_container *cont;
+ struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
struct mlx5_flow_counter_pool *pool;
- uint32_t batch = 0;
/* Decrease to original index and clear shared bit. */
idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
- if (IS_BATCH_CNT(idx)) {
- idx -= MLX5_CNT_BATCH_OFFSET;
- batch = 1;
- }
- cont = MLX5_CNT_CONTAINER(priv->sh, batch);
- MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
- pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
+ MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
+ pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
MLX5_ASSERT(pool);
if (ppool)
*ppool = pool;
@@ -4212,8 +4206,8 @@ struct field_modify_info modify_tcp[] = {
/**
* Get a pool by devx counter ID.
*
- * @param[in] cont
- * Pointer to the counter container.
+ * @param[in] cmng
+ * Pointer to the counter management.
* @param[in] id
* The counter devx ID.
*
@@ -4221,25 +4215,25 @@ struct field_modify_info modify_tcp[] = {
* The counter pool pointer if exists, NULL otherwise,
*/
static struct mlx5_flow_counter_pool *
-flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
+flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
{
uint32_t i;
/* Check last used pool. */
- if (cont->last_pool_idx != POOL_IDX_INVALID &&
- flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
- return cont->pools[cont->last_pool_idx];
+ if (cmng->last_pool_idx != POOL_IDX_INVALID &&
+ flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id))
+ return cmng->pools[cmng->last_pool_idx];
/* ID out of range means no suitable pool in the container. */
- if (id > cont->max_id || id < cont->min_id)
+ if (id > cmng->max_id || id < cmng->min_id)
return NULL;
/*
* Find the pool from the end of the container, since mostly counter
* ID is sequence increasing, and the last pool should be the needed
* one.
*/
- i = rte_atomic16_read(&cont->n_valid);
+ i = rte_atomic16_read(&cmng->n_valid);
while (i--) {
- struct mlx5_flow_counter_pool *pool = cont->pools[i];
+ struct mlx5_flow_counter_pool *pool = cmng->pools[i];
if (flow_dv_is_counter_in_pool(pool, id))
return pool;
@@ -4329,20 +4323,18 @@ struct field_modify_info modify_tcp[] = {
*
* @param[in] dev
* Pointer to the Ethernet device structure.
- * @param[in] batch
- * Whether the pool is for counter that was allocated by batch command.
*
* @return
* 0 on success, otherwise negative errno value and rte_errno is set.
*/
static int
-flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
+flow_dv_container_resize(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch);
+ struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
- void *old_pools = cont->pools;
- uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
+ void *old_pools = cmng->pools;
+ uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
@@ -4351,7 +4343,7 @@ struct field_modify_info modify_tcp[] = {
return -ENOMEM;
}
if (old_pools)
- memcpy(pools, old_pools, cont->n *
+ memcpy(pools, old_pools, cmng->n *
sizeof(struct mlx5_flow_counter_pool *));
/*
* Fallback mode query the counter directly, no background query
@@ -4372,11 +4364,11 @@ struct field_modify_info modify_tcp[] = {
MLX5_CNT_CONTAINER_RESIZE +
i, next);
}
- rte_spinlock_lock(&cont->resize_sl);
- cont->n = resize;
- cont->mem_mng = mem_mng;
- cont->pools = pools;
- rte_spinlock_unlock(&cont->resize_sl);
+ rte_spinlock_lock(&cmng->resize_sl);
+ cmng->n = resize;
+ cmng->mem_mng = mem_mng;
+ cmng->pools = pools;
+ rte_spinlock_unlock(&cmng->resize_sl);
if (old_pools)
mlx5_free(old_pools);
return 0;
@@ -4409,27 +4401,16 @@ struct field_modify_info modify_tcp[] = {
cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
MLX5_ASSERT(pool);
- if (!IS_BATCH_CNT(counter)) {
+ if (priv->counter_fallback) {
cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
- if (priv->counter_fallback)
- return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
+ return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
0, pkts, bytes, 0, NULL, NULL, 0);
}
rte_spinlock_lock(&pool->sl);
- /*
- * The single counters allocation may allocate smaller ID than the
- * current allocated in parallel to the host reading.
- * In this case the new counter values must be reported as 0.
- */
- if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
- *pkts = 0;
- *bytes = 0;
- } else {
- offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
- *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
- *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
- }
+ offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
+ *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
+ *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
rte_spinlock_unlock(&pool->sl);
return 0;
}
@@ -4441,8 +4422,6 @@ struct field_modify_info modify_tcp[] = {
* Pointer to the Ethernet device structure.
* @param[out] dcs
* The devX counter handle.
- * @param[in] batch
- * Whether the pool is for counter that was allocated by batch command.
* @param[in] age
* Whether the pool is for counter that was allocated for aging.
* @param[in/out] cont_cur
@@ -4453,123 +4432,63 @@ struct field_modify_info modify_tcp[] = {
*/
static struct mlx5_flow_counter_pool *
flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
- uint32_t batch, uint32_t age)
+ uint32_t age)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_counter_pool *pool;
- struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch);
- int16_t n_valid = rte_atomic16_read(&cont->n_valid);
+ struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
+ int16_t n_valid = rte_atomic16_read(&cmng->n_valid);
+ uint32_t fallback = priv->counter_fallback;
uint32_t size = sizeof(*pool);
- if (cont->n == n_valid && flow_dv_container_resize(dev, batch))
+ if (cmng->n == n_valid && flow_dv_container_resize(dev))
return NULL;
size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
- size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
+ size += (!fallback ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
if (!pool) {
rte_errno = ENOMEM;
return NULL;
}
- pool->min_dcs = dcs;
- if (!priv->counter_fallback)
- pool->raw = cont->mem_mng->raws + n_valid %
+ if (!fallback) {
+ pool->min_dcs = dcs;
+ pool->raw = cmng->mem_mng->raws + n_valid %
MLX5_CNT_CONTAINER_RESIZE;
+ }
pool->raw_hw = NULL;
pool->type = 0;
- pool->type |= (batch ? 0 : CNT_POOL_TYPE_EXT);
+ pool->type |= (!fallback ? 0 : CNT_POOL_TYPE_EXT);
pool->type |= (!age ? 0 : CNT_POOL_TYPE_AGE);
pool->query_gen = 0;
rte_spinlock_init(&pool->sl);
TAILQ_INIT(&pool->counters[0]);
TAILQ_INIT(&pool->counters[1]);
- TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
+ TAILQ_INSERT_HEAD(&cmng->pool_list, pool, next);
pool->index = n_valid;
- cont->pools[n_valid] = pool;
- if (!batch) {
+ cmng->pools[n_valid] = pool;
+ if (fallback) {
int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
- if (base < cont->min_id)
- cont->min_id = base;
- if (base > cont->max_id)
- cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
- cont->last_pool_idx = pool->index;
+ if (base < cmng->min_id)
+ cmng->min_id = base;
+ if (base > cmng->max_id)
+ cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
+ cmng->last_pool_idx = pool->index;
}
/* Pool initialization must be updated before host thread access. */
rte_io_wmb();
- rte_atomic16_add(&cont->n_valid, 1);
+ rte_atomic16_add(&cmng->n_valid, 1);
return pool;
}
/**
- * Restore skipped counters in the pool.
- *
- * As counter pool query requires the first counter dcs
- * ID start with 4 alinged, if the pool counters with
- * min_dcs ID are not aligned with 4, the counters will
- * be skipped.
- * Once other min_dcs ID less than these skipped counter
- * dcs ID appears, the skipped counters will be safe to
- * use.
- * Should be called when min_dcs is updated.
- *
- * @param[in] pool
- * Current counter pool.
- * @param[in] last_min_dcs
- * Last min_dcs.
- */
-static void
-flow_dv_counter_restore(struct mlx5_flow_counter_pool *pool,
- struct mlx5_devx_obj *last_min_dcs)
-{
- struct mlx5_flow_counter_ext *cnt_ext;
- uint32_t offset, new_offset;
- uint32_t skip_cnt = 0;
- uint32_t i;
-
- if (!pool->skip_cnt)
- return;
- /*
- * If last min_dcs is not valid. The skipped counter may even after
- * last min_dcs, set the offset to the whole pool.
- */
- if (last_min_dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
- offset = MLX5_COUNTERS_PER_POOL;
- else
- offset = last_min_dcs->id % MLX5_COUNTERS_PER_POOL;
- new_offset = pool->min_dcs->id % MLX5_COUNTERS_PER_POOL;
- /*
- * Check the counters from 1 to the last_min_dcs range. Counters
- * before new min_dcs indicates pool still has skipped counters.
- * Counters be skipped after new min_dcs will be ready to use.
- * Offset 0 counter must be empty or min_dcs, start from 1.
- */
- for (i = 1; i < offset; i++) {
- cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
- if (cnt_ext->skipped) {
- if (i > new_offset) {
- cnt_ext->skipped = 0;
- TAILQ_INSERT_TAIL
- (&pool->counters[pool->query_gen],
- MLX5_POOL_GET_CNT(pool, i), next);
- } else {
- skip_cnt++;
- }
- }
- }
- if (!skip_cnt)
- pool->skip_cnt = 0;
-}
-
-/**
* Prepare a new counter and/or a new counter pool.
*
* @param[in] dev
* Pointer to the Ethernet device structure.
* @param[out] cnt_free
* Where to put the pointer of a new counter.
- * @param[in] batch
- * Whether the pool is for counter that was allocated by batch command.
* @param[in] age
* Whether the pool is for counter that was allocated for aging.
*
@@ -4580,87 +4499,36 @@ struct field_modify_info modify_tcp[] = {
static struct mlx5_flow_counter_pool *
flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
struct mlx5_flow_counter **cnt_free,
- uint32_t batch, uint32_t age)
+ uint32_t age)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_pools_container *cont;
+ struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
struct mlx5_flow_counter_pool *pool;
struct mlx5_counters tmp_tq;
- struct mlx5_devx_obj *last_min_dcs;
struct mlx5_devx_obj *dcs = NULL;
struct mlx5_flow_counter *cnt;
enum mlx5_counter_type cnt_type =
age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
- uint32_t add2other;
+ uint32_t fallback = priv->counter_fallback;
uint32_t i;
- cont = MLX5_CNT_CONTAINER(priv->sh, batch);
- if (!batch) {
-retry:
- add2other = 0;
+ if (fallback) {
/* bulk_bitmap must be 0 for single counter allocation. */
dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
if (!dcs)
return NULL;
- pool = flow_dv_find_pool_by_id(cont, dcs->id);
- /*
- * If pool eixsts but with other type, counter will be added
- * to the other pool, need to reallocate new counter in the
- * ragne with same type later.
- */
- if (pool && ((!!IS_AGE_POOL(pool)) != age)) {
- add2other = 1;
- } else if (!pool) {
- pool = flow_dv_pool_create(dev, dcs, batch,
- age);
+ pool = flow_dv_find_pool_by_id(cmng, dcs->id);
+ if (!pool) {
+ pool = flow_dv_pool_create(dev, dcs, age);
if (!pool) {
mlx5_devx_cmd_destroy(dcs);
return NULL;
}
}
- if ((dcs->id < pool->min_dcs->id ||
- pool->min_dcs->id &
- (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) &&
- !(dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))) {
- /*
- * Update the pool min_dcs only if current dcs is
- * valid and exist min_dcs is not valid or greater
- * than new dcs.
- */
- last_min_dcs = pool->min_dcs;
- rte_atomic64_set(&pool->a64_dcs,
- (int64_t)(uintptr_t)dcs);
- /*
- * Restore any skipped counters if the new min_dcs
- * ID is smaller or min_dcs is not valid.
- */
- if (dcs->id < last_min_dcs->id ||
- last_min_dcs->id &
- (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
- flow_dv_counter_restore(pool, last_min_dcs);
- }
i = dcs->id % MLX5_COUNTERS_PER_POOL;
cnt = MLX5_POOL_GET_CNT(pool, i);
cnt->pool = pool;
MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
- /*
- * If min_dcs is not valid, it means the new allocated dcs
- * also fail to become the valid min_dcs, just skip it.
- * Or if min_dcs is valid, and new dcs ID is smaller than
- * min_dcs, but not become the min_dcs, also skip it.
- */
- if (pool->min_dcs->id &
- (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1) ||
- dcs->id < pool->min_dcs->id) {
- MLX5_GET_POOL_CNT_EXT(pool, i)->skipped = 1;
- pool->skip_cnt = 1;
- goto retry;
- }
- if (add2other) {
- TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen],
- cnt, next);
- goto retry;
- }
*cnt_free = cnt;
return pool;
}
@@ -4671,7 +4539,7 @@ struct field_modify_info modify_tcp[] = {
rte_errno = ENODATA;
return NULL;
}
- pool = flow_dv_pool_create(dev, dcs, batch, age);
+ pool = flow_dv_pool_create(dev, dcs, age);
if (!pool) {
mlx5_devx_cmd_destroy(dcs);
return NULL;
@@ -4682,9 +4550,9 @@ struct field_modify_info modify_tcp[] = {
cnt->pool = pool;
TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
}
- rte_spinlock_lock(&cont->csl);
- TAILQ_CONCAT(&cont->counters[cnt_type], &tmp_tq, next);
- rte_spinlock_unlock(&cont->csl);
+ rte_spinlock_lock(&cmng->csl);
+ TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
+ rte_spinlock_unlock(&cmng->csl);
*cnt_free = MLX5_POOL_GET_CNT(pool, 0);
(*cnt_free)->pool = pool;
return pool;
@@ -4721,8 +4589,6 @@ struct field_modify_info modify_tcp[] = {
* Indicate if this counter is shared with other flows.
* @param[in] id
* Counter identifier.
- * @param[in] group
- * Counter flow group.
* @param[in] age
* Whether the counter was allocated for aging.
*
@@ -4731,22 +4597,14 @@ struct field_modify_info modify_tcp[] = {
*/
static uint32_t
flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
- uint16_t group, uint32_t age)
+ uint32_t age)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_counter_pool *pool = NULL;
struct mlx5_flow_counter *cnt_free = NULL;
struct mlx5_flow_counter_ext *cnt_ext = NULL;
- /*
- * Currently group 0 flow counter cannot be assigned to a flow if it is
- * not the first one in the batch counter allocation, so it is better
- * to allocate counters one by one for these flows in a separate
- * container.
- * A counter can be shared between different groups so need to take
- * shared counters from the single container.
- */
- uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
- struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch);
+ uint32_t fallback = priv->counter_fallback;
+ struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
enum mlx5_counter_type cnt_type =
age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
uint32_t cnt_idx;
@@ -4769,16 +4627,15 @@ struct field_modify_info modify_tcp[] = {
}
}
/* Get free counters from container. */
- rte_spinlock_lock(&cont->csl);
- cnt_free = TAILQ_FIRST(&cont->counters[cnt_type]);
+ rte_spinlock_lock(&cmng->csl);
+ cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
if (cnt_free)
- TAILQ_REMOVE(&cont->counters[cnt_type], cnt_free, next);
- rte_spinlock_unlock(&cont->csl);
- if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
- batch, age))
+ TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
+ rte_spinlock_unlock(&cmng->csl);
+ if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
goto err;
pool = cnt_free->pool;
- if (!batch)
+ if (fallback)
cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
/* Create a DV counter action only in the first time usage. */
if (!cnt_free->action) {
@@ -4786,7 +4643,7 @@ struct field_modify_info modify_tcp[] = {
struct mlx5_devx_obj *dcs;
int ret;
- if (batch) {
+ if (!fallback) {
offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
dcs = pool->min_dcs;
} else {
@@ -4802,7 +4659,6 @@ struct field_modify_info modify_tcp[] = {
}
cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
MLX5_CNT_ARRAY_IDX(pool, cnt_free));
- cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
/* Update the counter reset values. */
if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
&cnt_free->bytes))
@@ -4817,16 +4673,16 @@ struct field_modify_info modify_tcp[] = {
cnt_free->shared_info.id = id;
cnt_idx |= MLX5_CNT_SHARED_OFFSET;
}
- if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
+ if (!fallback && !priv->sh->cmng.query_thread_on)
/* Start the asynchronous batch query by the host thread. */
mlx5_set_query_alarm(priv->sh);
return cnt_idx;
err:
if (cnt_free) {
cnt_free->pool = pool;
- rte_spinlock_lock(&cont->csl);
- TAILQ_INSERT_TAIL(&cont->counters[cnt_type], cnt_free, next);
- rte_spinlock_unlock(&cont->csl);
+ rte_spinlock_lock(&cmng->csl);
+ TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
+ rte_spinlock_unlock(&cmng->csl);
}
return 0;
}
@@ -4909,7 +4765,6 @@ struct field_modify_info modify_tcp[] = {
return;
cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
MLX5_ASSERT(pool);
-
if (IS_SHARED_CNT(counter)) {
if (--cnt->shared_info.ref_cnt)
return;
@@ -4934,13 +4789,43 @@ struct field_modify_info modify_tcp[] = {
} else {
cnt_type = IS_AGE_POOL(pool) ? MLX5_COUNTER_TYPE_AGE :
MLX5_COUNTER_TYPE_ORIGIN;
- TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
- (priv->sh, 0))->counters[cnt_type]),
+ TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
cnt, next);
}
}
/**
+ * Create a counter action with invalid offset.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ *
+ * @return
+ * Counter action pointer if success, NULL otherwise.
+ */
+static void*
+flow_dv_counter_create_invalid(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ int ret;
+
+ if (!priv->cnt_dcs) {
+ priv->cnt_dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx,
+ 0x4);
+ if (!priv->cnt_dcs)
+ return NULL;
+ }
+ if (!priv->cnt_action) {
+ ret = mlx5_flow_os_create_flow_action_count(priv->cnt_dcs->obj,
+ UINT16_MAX,
+ &priv->cnt_action);
+ if (ret)
+ return NULL;
+ }
+ return priv->cnt_action;
+}
+
+/**
* Verify the @p attributes will be correctly understood by the NIC and store
* them in the @p flow if everything is correct.
*
@@ -5781,6 +5666,8 @@ struct field_modify_info modify_tcp[] = {
action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
rw_act_num += MLX5_ACT_NUM_SET_DSCP;
break;
+ case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
+ break;
default:
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION,
@@ -7988,8 +7875,7 @@ struct field_modify_info modify_tcp[] = {
counter = flow_dv_counter_alloc(dev,
count ? count->shared : 0,
- count ? count->id : 0,
- dev_flow->dv.group, !!age);
+ count ? count->id : 0, !!age);
if (!counter || age == NULL)
return counter;
age_param = flow_dv_counter_idx_get_age(dev, counter);
@@ -8359,6 +8245,13 @@ struct field_modify_info modify_tcp[] = {
age = action->conf;
action_flags |= MLX5_FLOW_ACTION_COUNT;
break;
+ case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
+ if (flow_dv_counter_create_invalid(dev) == NULL)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "count action not supported");
+ dev_flow->dv.actions[actions_n++] = priv->cnt_action;
+ break;
case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
dev_flow->dv.actions[actions_n++] =
priv->sh->pop_vlan_action;
@@ -10112,7 +10005,7 @@ struct field_modify_info modify_tcp[] = {
uint32_t cnt;
flow_dv_shared_lock(dev);
- cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
+ cnt = flow_dv_counter_alloc(dev, 0, 0, 0);
flow_dv_shared_unlock(dev);
return cnt;
}
@@ -159,11 +159,11 @@
struct mlx5_flow_counter_pool **ppool)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0);
+ struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
struct mlx5_flow_counter_pool *pool;
idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
- pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
+ pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
MLX5_ASSERT(pool);
if (ppool)
*ppool = pool;
@@ -254,12 +254,12 @@
flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0);
+ struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
struct mlx5_flow_counter_pool *pool = NULL;
struct mlx5_flow_counter_ext *cnt_ext = NULL;
struct mlx5_flow_counter *cnt = NULL;
union mlx5_l3t_data data;
- uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
+ uint32_t n_valid = rte_atomic16_read(&cmng->n_valid);
uint32_t pool_idx, cnt_idx;
uint32_t i;
int ret;
@@ -275,7 +275,7 @@
return data.dword;
}
for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
- pool = cont->pools[pool_idx];
+ pool = cmng->pools[pool_idx];
if (!pool)
continue;
cnt = TAILQ_FIRST(&pool->counters[0]);
@@ -286,7 +286,7 @@
struct mlx5_flow_counter_pool **pools;
uint32_t size;
- if (n_valid == cont->n) {
+ if (n_valid == cmng->n) {
/* Resize the container pool array. */
size = sizeof(struct mlx5_flow_counter_pool *) *
(n_valid + MLX5_CNT_CONTAINER_RESIZE);
@@ -295,13 +295,13 @@
if (!pools)
return 0;
if (n_valid) {
- memcpy(pools, cont->pools,
+ memcpy(pools, cmng->pools,
sizeof(struct mlx5_flow_counter_pool *) *
n_valid);
- mlx5_free(cont->pools);
+ mlx5_free(cmng->pools);
}
- cont->pools = pools;
- cont->n += MLX5_CNT_CONTAINER_RESIZE;
+ cmng->pools = pools;
+ cmng->n += MLX5_CNT_CONTAINER_RESIZE;
}
/* Allocate memory for new pool*/
size = sizeof(*pool) + (sizeof(*cnt_ext) + sizeof(*cnt)) *
@@ -315,10 +315,10 @@
TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
}
cnt = MLX5_POOL_GET_CNT(pool, 0);
- cont->pools[n_valid] = pool;
+ cmng->pools[n_valid] = pool;
pool_idx = n_valid;
- rte_atomic16_add(&cont->n_valid, 1);
- TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
+ rte_atomic16_add(&cmng->n_valid, 1);
+ TAILQ_INSERT_HEAD(&cmng->pool_list, pool, next);
}
i = MLX5_CNT_ARRAY_IDX(pool, cnt);
cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i);