@@ -465,8 +465,7 @@ struct mlx5_flow_id_pool *
sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET;
sh->cmng.max_id = -1;
sh->cmng.last_pool_idx = POOL_IDX_INVALID;
- TAILQ_INIT(&sh->cmng.pool_list);
- rte_spinlock_init(&sh->cmng.resize_sl);
+ rte_spinlock_init(&sh->cmng.pool_update_sl);
for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++)
TAILQ_INIT(&sh->cmng.counters[i]);
rte_spinlock_init(&sh->cmng.csl);
@@ -499,7 +498,7 @@ struct mlx5_flow_id_pool *
mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh)
{
struct mlx5_counter_stats_mem_mng *mng;
- int j;
+ int i, j;
int retries = 1024;
rte_errno = 0;
@@ -512,9 +511,10 @@ struct mlx5_flow_id_pool *
if (sh->cmng.pools) {
struct mlx5_flow_counter_pool *pool;
+ int n_valid = sh->cmng.n_valid;
- pool = TAILQ_FIRST(&sh->cmng.pool_list);
- while (pool) {
+ for (i = 0; i < n_valid; ++i) {
+ pool = sh->cmng.pools[i];
if (!IS_EXT_POOL(pool) && pool->min_dcs)
claim_zero(mlx5_devx_cmd_destroy
(pool->min_dcs));
@@ -530,9 +530,7 @@ struct mlx5_flow_id_pool *
(MLX5_GET_POOL_CNT_EXT
(pool, j)->dcs));
}
- TAILQ_REMOVE(&sh->cmng.pool_list, pool, next);
mlx5_free(pool);
- pool = TAILQ_FIRST(&sh->cmng.pool_list);
}
mlx5_free(sh->cmng.pools);
}
@@ -395,7 +395,11 @@ struct mlx5_flow_counter_pool {
volatile uint32_t query_gen:1; /* Query round. */
rte_spinlock_t sl; /* The pool lock. */
struct mlx5_counter_stats_raw *raw;
- struct mlx5_counter_stats_raw *raw_hw; /* The raw on HW working. */
+ union {
+ struct rte_eth_dev *dev; /* The counter pool create device. */
+ struct mlx5_counter_stats_raw *raw_hw;
+ /* The raw on HW working. */
+ };
};
struct mlx5_counter_stats_raw;
@@ -419,16 +423,15 @@ struct mlx5_counter_stats_raw {
/* Counter global management structure. */
struct mlx5_flow_counter_mng {
- rte_atomic16_t n_valid; /* Number of valid pools. */
+ volatile uint16_t n_valid; /* Number of valid pools. */
uint16_t n; /* Number of pools. */
uint16_t last_pool_idx; /* Last used pool index */
int min_id; /* The minimum counter ID in the pools. */
int max_id; /* The maximum counter ID in the pools. */
- rte_spinlock_t resize_sl; /* The resize lock. */
+ rte_spinlock_t pool_update_sl; /* The pool update lock. */
rte_spinlock_t csl; /* The counter free list lock. */
struct mlx5_counters counters[MLX5_COUNTER_TYPE_MAX];
/* Free counter list. */
- struct mlx5_counter_pools pool_list; /* Counter pool list. */
struct mlx5_flow_counter_pool **pools; /* Counter pool array. */
struct mlx5_counter_stats_mem_mng *mem_mng;
/* Hold the memory management for the next allocated pools raws. */
@@ -13,6 +13,7 @@
#include <rte_common.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
+#include <rte_eal_paging.h>
#include <rte_flow.h>
#include <rte_cycles.h>
#include <rte_flow_driver.h>
@@ -29,6 +30,7 @@
#include "mlx5_flow.h"
#include "mlx5_flow_os.h"
#include "mlx5_rxtx.h"
+#include "mlx5_common_os.h"
/** Device flow drivers. */
extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
@@ -5880,6 +5882,116 @@ struct mlx5_meter_domains_infos *
return -ENOTSUP;
}
+/**
+ * Allocate a new memory for the counter values wrapped by all the needed
+ * management.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in] raws_n
+ * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise.
+ */
+static int
+mlx5_flow_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_dev_ctx_shared *sh = priv->sh;
+ struct mlx5_devx_mkey_attr mkey_attr;
+ struct mlx5_counter_stats_mem_mng *mem_mng;
+ volatile struct flow_counter_stats *raw_data;
+ int size = (sizeof(struct flow_counter_stats) *
+ MLX5_COUNTERS_PER_POOL +
+ sizeof(struct mlx5_counter_stats_raw)) * raws_n +
+ sizeof(struct mlx5_counter_stats_mem_mng);
+ size_t pgsize = rte_mem_page_size();
+ if (pgsize == (size_t)-1) {
+ DRV_LOG(ERR, "Failed to get mem page size");
+ rte_errno = ENOMEM;
+ return -ENOMEM;
+ }
+ uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
+ SOCKET_ID_ANY);
+ int i;
+
+ if (!mem) {
+ rte_errno = ENOMEM;
+ return -ENOMEM;
+ }
+ mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
+ size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
+ mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!mem_mng->umem) {
+ rte_errno = errno;
+ mlx5_free(mem);
+ return -rte_errno;
+ }
+ mkey_attr.addr = (uintptr_t)mem;
+ mkey_attr.size = size;
+ mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
+ mkey_attr.pd = sh->pdn;
+ mkey_attr.log_entity_size = 0;
+ mkey_attr.pg_access = 0;
+ mkey_attr.klm_array = NULL;
+ mkey_attr.klm_num = 0;
+ if (priv->config.hca_attr.relaxed_ordering_write &&
+ priv->config.hca_attr.relaxed_ordering_read &&
+ !haswell_broadwell_cpu)
+ mkey_attr.relaxed_ordering = 1;
+ mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
+ if (!mem_mng->dm) {
+ mlx5_glue->devx_umem_dereg(mem_mng->umem);
+ rte_errno = errno;
+ mlx5_free(mem);
+ return -rte_errno;
+ }
+ mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
+ raw_data = (volatile struct flow_counter_stats *)mem;
+ for (i = 0; i < raws_n; ++i) {
+ mem_mng->raws[i].mem_mng = mem_mng;
+ mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
+ }
+ for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
+ LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
+ mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
+ next);
+ LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
+ priv->sh->cmng.mem_mng = mem_mng;
+ return 0;
+}
+
+/**
+ * Set the statistic memory to the new counter pool.
+ *
+ * @param[in] cmng
+ * Pointer to the counter management.
+ * @param[in] pool
+ * Pointer to the pool to set the statistic memory.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise.
+ */
+static int
+mlx5_flow_set_counter_stat_mem(struct mlx5_flow_counter_mng *cmng,
+ struct mlx5_flow_counter_pool *pool)
+{
+ /* Resize statistic memory once used out. */
+ if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
+ mlx5_flow_create_counter_stat_mem_mng(pool->dev,
+ MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES)) {
+ DRV_LOG(ERR, "Cannot resize counter stat mem.");
+ return -1;
+ }
+ MLX5_ASSERT(pool->index < n_valid);
+ pool->raw = cmng->mem_mng->raws + pool->index %
+ MLX5_CNT_CONTAINER_RESIZE;
+ pool->raw_hw = NULL;
+ return 0;
+}
+
#define MLX5_POOL_QUERY_FREQ_US 1000000
/**
@@ -5894,7 +6006,7 @@ struct mlx5_meter_domains_infos *
{
uint32_t pools_n, us;
- pools_n = rte_atomic16_read(&sh->cmng.n_valid);
+ pools_n = sh->cmng.n_valid;
us = MLX5_POOL_QUERY_FREQ_US / pools_n;
DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
@@ -5920,16 +6032,21 @@ struct mlx5_meter_domains_infos *
uint16_t pool_index = sh->cmng.pool_index;
struct mlx5_flow_counter_mng *cmng = &sh->cmng;
struct mlx5_flow_counter_pool *pool;
+ int n_valid;
if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
goto set_alarm;
- rte_spinlock_lock(&cmng->resize_sl);
+ rte_spinlock_lock(&cmng->pool_update_sl);
if (!cmng->pools) {
- rte_spinlock_unlock(&cmng->resize_sl);
+ rte_spinlock_unlock(&cmng->pool_update_sl);
goto set_alarm;
}
pool = cmng->pools[pool_index];
- rte_spinlock_unlock(&cmng->resize_sl);
+ n_valid = cmng->n_valid;
+ rte_spinlock_unlock(&cmng->pool_update_sl);
+ /* Set the statistic memory to the new created pool. */
+ if ((!pool->raw && mlx5_flow_set_counter_stat_mem(cmng, pool)))
+ goto set_alarm;
if (pool->raw_hw)
/* There is a pool query in progress. */
goto set_alarm;
@@ -5962,7 +6079,7 @@ struct mlx5_meter_domains_infos *
LIST_REMOVE(pool->raw_hw, next);
sh->cmng.pending_queries++;
pool_index++;
- if (pool_index >= rte_atomic16_read(&cmng->n_valid))
+ if (pool_index >= n_valid)
pool_index = 0;
set_alarm:
sh->cmng.pool_index = pool_index;
@@ -4231,7 +4231,7 @@ struct field_modify_info modify_tcp[] = {
* ID is sequence increasing, and the last pool should be the needed
* one.
*/
- i = rte_atomic16_read(&cmng->n_valid);
+ i = cmng->n_valid;
while (i--) {
struct mlx5_flow_counter_pool *pool = cmng->pools[i];
@@ -4242,83 +4242,6 @@ struct field_modify_info modify_tcp[] = {
}
/**
- * Allocate a new memory for the counter values wrapped by all the needed
- * management.
- *
- * @param[in] dev
- * Pointer to the Ethernet device structure.
- * @param[in] raws_n
- * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
- *
- * @return
- * The new memory management pointer on success, otherwise NULL and rte_errno
- * is set.
- */
-static struct mlx5_counter_stats_mem_mng *
-flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_dev_ctx_shared *sh = priv->sh;
- struct mlx5_devx_mkey_attr mkey_attr;
- struct mlx5_counter_stats_mem_mng *mem_mng;
- volatile struct flow_counter_stats *raw_data;
- int size = (sizeof(struct flow_counter_stats) *
- MLX5_COUNTERS_PER_POOL +
- sizeof(struct mlx5_counter_stats_raw)) * raws_n +
- sizeof(struct mlx5_counter_stats_mem_mng);
- size_t pgsize = rte_mem_page_size();
- if (pgsize == (size_t)-1) {
- DRV_LOG(ERR, "Failed to get mem page size");
- rte_errno = ENOMEM;
- return NULL;
- }
- uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
- SOCKET_ID_ANY);
- int i;
-
- if (!mem) {
- rte_errno = ENOMEM;
- return NULL;
- }
- mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
- size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
- mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
- IBV_ACCESS_LOCAL_WRITE);
- if (!mem_mng->umem) {
- rte_errno = errno;
- mlx5_free(mem);
- return NULL;
- }
- mkey_attr.addr = (uintptr_t)mem;
- mkey_attr.size = size;
- mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
- mkey_attr.pd = sh->pdn;
- mkey_attr.log_entity_size = 0;
- mkey_attr.pg_access = 0;
- mkey_attr.klm_array = NULL;
- mkey_attr.klm_num = 0;
- if (priv->config.hca_attr.relaxed_ordering_write &&
- priv->config.hca_attr.relaxed_ordering_read &&
- !haswell_broadwell_cpu)
- mkey_attr.relaxed_ordering = 1;
- mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
- if (!mem_mng->dm) {
- mlx5_glue->devx_umem_dereg(mem_mng->umem);
- rte_errno = errno;
- mlx5_free(mem);
- return NULL;
- }
- mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
- raw_data = (volatile struct flow_counter_stats *)mem;
- for (i = 0; i < raws_n; ++i) {
- mem_mng->raws[i].mem_mng = mem_mng;
- mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
- }
- LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
- return mem_mng;
-}
-
-/**
* Resize a counter container.
*
* @param[in] dev
@@ -4332,7 +4255,6 @@ struct field_modify_info modify_tcp[] = {
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
- struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
void *old_pools = cmng->pools;
uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
@@ -4345,30 +4267,8 @@ struct field_modify_info modify_tcp[] = {
if (old_pools)
memcpy(pools, old_pools, cmng->n *
sizeof(struct mlx5_flow_counter_pool *));
- /*
- * Fallback mode query the counter directly, no background query
- * resources are needed.
- */
- if (!priv->counter_fallback) {
- int i;
-
- mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
- MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
- if (!mem_mng) {
- mlx5_free(pools);
- return -ENOMEM;
- }
- for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
- LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
- mem_mng->raws +
- MLX5_CNT_CONTAINER_RESIZE +
- i, next);
- }
- rte_spinlock_lock(&cmng->resize_sl);
cmng->n = resize;
- cmng->mem_mng = mem_mng;
cmng->pools = pools;
- rte_spinlock_unlock(&cmng->resize_sl);
if (old_pools)
mlx5_free(old_pools);
return 0;
@@ -4406,11 +4306,15 @@ struct field_modify_info modify_tcp[] = {
return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
0, pkts, bytes, 0, NULL, NULL, 0);
}
-
rte_spinlock_lock(&pool->sl);
- offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
- *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
- *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
+ if (!pool->raw) {
+ *pkts = 0;
+ *bytes = 0;
+ } else {
+ offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
+ *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
+ *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
+ }
rte_spinlock_unlock(&pool->sl);
return 0;
}
@@ -4437,12 +4341,9 @@ struct field_modify_info modify_tcp[] = {
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_counter_pool *pool;
struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
- int16_t n_valid = rte_atomic16_read(&cmng->n_valid);
uint32_t fallback = priv->counter_fallback;
uint32_t size = sizeof(*pool);
- if (cmng->n == n_valid && flow_dv_container_resize(dev))
- return NULL;
size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
size += (!fallback ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
@@ -4451,23 +4352,25 @@ struct field_modify_info modify_tcp[] = {
rte_errno = ENOMEM;
return NULL;
}
- if (!fallback) {
- pool->min_dcs = dcs;
- pool->raw = cmng->mem_mng->raws + n_valid %
- MLX5_CNT_CONTAINER_RESIZE;
- }
- pool->raw_hw = NULL;
+ pool->raw = NULL;
pool->type = 0;
- pool->type |= (!fallback ? 0 : CNT_POOL_TYPE_EXT);
pool->type |= (!age ? 0 : CNT_POOL_TYPE_AGE);
pool->query_gen = 0;
+ pool->min_dcs = dcs;
rte_spinlock_init(&pool->sl);
TAILQ_INIT(&pool->counters[0]);
TAILQ_INIT(&pool->counters[1]);
- TAILQ_INSERT_HEAD(&cmng->pool_list, pool, next);
- pool->index = n_valid;
- cmng->pools[n_valid] = pool;
- if (fallback) {
+ rte_spinlock_lock(&cmng->pool_update_sl);
+ pool->index = cmng->n_valid;
+ if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
+ mlx5_free(pool);
+ rte_spinlock_unlock(&cmng->pool_update_sl);
+ return NULL;
+ }
+ cmng->pools[pool->index] = pool;
+ pool->dev = dev;
+ cmng->n_valid++;
+ if (unlikely(fallback)) {
int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
if (base < cmng->min_id)
@@ -4475,10 +4378,9 @@ struct field_modify_info modify_tcp[] = {
if (base > cmng->max_id)
cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
cmng->last_pool_idx = pool->index;
+ pool->type |= CNT_POOL_TYPE_EXT;
}
- /* Pool initialization must be updated before host thread access. */
- rte_io_wmb();
- rte_atomic16_add(&cmng->n_valid, 1);
+ rte_spinlock_unlock(&cmng->pool_update_sl);
return pool;
}
@@ -259,7 +259,7 @@
struct mlx5_flow_counter_ext *cnt_ext = NULL;
struct mlx5_flow_counter *cnt = NULL;
union mlx5_l3t_data data;
- uint32_t n_valid = rte_atomic16_read(&cmng->n_valid);
+ uint32_t n_valid = cmng->n_valid;
uint32_t pool_idx, cnt_idx;
uint32_t i;
int ret;
@@ -317,8 +317,7 @@
cnt = MLX5_POOL_GET_CNT(pool, 0);
cmng->pools[n_valid] = pool;
pool_idx = n_valid;
- rte_atomic16_add(&cmng->n_valid, 1);
- TAILQ_INSERT_HEAD(&cmng->pool_list, pool, next);
+ cmng->n_valid++;
}
i = MLX5_CNT_ARRAY_IDX(pool, cnt);
cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i);