From patchwork Wed Mar 31 07:36:19 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90171 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 36A4FA034F; Wed, 31 Mar 2021 09:36:49 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id A79E4140DE4; Wed, 31 Mar 2021 09:36:44 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id B3BB04069E for ; Wed, 31 Mar 2021 09:36:41 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:40 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adw9001778; Wed, 31 Mar 2021 10:36:40 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com, Suanming Mou Date: Wed, 31 Mar 2021 10:36:19 +0300 Message-Id: <20210331073632.1443011-2-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 01/13] net/mlx5: support three level table walk X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Suanming Mou This commit adds table entry walk for the three level table. Signed-off-by: Suanming Mou --- drivers/net/mlx5/mlx5_utils.h | 90 +++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h index 7a62187f8e..f6703391c6 100644 --- a/drivers/net/mlx5/mlx5_utils.h +++ b/drivers/net/mlx5/mlx5_utils.h @@ -832,6 +832,91 @@ int32_t mlx5_l3t_clear_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx); int32_t mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx, union mlx5_l3t_data *data); +static inline void * +mlx5_l3t_get_next(struct mlx5_l3t_tbl *tbl, uint32_t *pos) +{ + struct mlx5_l3t_level_tbl *g_tbl, *m_tbl; + uint32_t i, j, k, g_start, m_start, e_start; + uint32_t idx = *pos; + void *e_tbl; + struct mlx5_l3t_entry_word *w_e_tbl; + struct mlx5_l3t_entry_dword *dw_e_tbl; + struct mlx5_l3t_entry_qword *qw_e_tbl; + struct mlx5_l3t_entry_ptr *ptr_e_tbl; + + if (!tbl) + return NULL; + g_tbl = tbl->tbl; + if (!g_tbl) + return NULL; + g_start = (idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK; + m_start = (idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK; + e_start = idx & MLX5_L3T_ET_MASK; + for (i = g_start; i < MLX5_L3T_GT_SIZE; i++) { + m_tbl = g_tbl->tbl[i]; + if (!m_tbl) { + /* Jump to new table, reset the sub table start. */ + m_start = 0; + e_start = 0; + continue; + } + for (j = m_start; j < MLX5_L3T_MT_SIZE; j++) { + if (!m_tbl->tbl[j]) { + /* + * Jump to new table, reset the sub table + * start. + */ + e_start = 0; + continue; + } + e_tbl = m_tbl->tbl[j]; + switch (tbl->type) { + case MLX5_L3T_TYPE_WORD: + w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl; + for (k = e_start; k < MLX5_L3T_ET_SIZE; k++) { + if (!w_e_tbl->entry[k].data) + continue; + *pos = (i << MLX5_L3T_GT_OFFSET) | + (j << MLX5_L3T_MT_OFFSET) | k; + return (void *)&w_e_tbl->entry[k].data; + } + break; + case MLX5_L3T_TYPE_DWORD: + dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl; + for (k = e_start; k < MLX5_L3T_ET_SIZE; k++) { + if (!dw_e_tbl->entry[k].data) + continue; + *pos = (i << MLX5_L3T_GT_OFFSET) | + (j << MLX5_L3T_MT_OFFSET) | k; + return (void *)&dw_e_tbl->entry[k].data; + } + break; + case MLX5_L3T_TYPE_QWORD: + qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl; + for (k = e_start; k < MLX5_L3T_ET_SIZE; k++) { + if (!qw_e_tbl->entry[k].data) + continue; + *pos = (i << MLX5_L3T_GT_OFFSET) | + (j << MLX5_L3T_MT_OFFSET) | k; + return (void *)&qw_e_tbl->entry[k].data; + } + break; + default: + ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl; + for (k = e_start; k < MLX5_L3T_ET_SIZE; k++) { + if (!ptr_e_tbl->entry[k].data) + continue; + *pos = (i << MLX5_L3T_GT_OFFSET) | + (j << MLX5_L3T_MT_OFFSET) | k; + return ptr_e_tbl->entry[k].data; + } + break; + } + } + } + return NULL; +} + /* * Macros for linked list based on indexed memory. * Example data structure: @@ -907,4 +992,9 @@ struct { \ idx = (elem)->field.next, (elem) = \ (idx) ? mlx5_ipool_get(pool, idx) : NULL) +#define MLX5_L3T_FOREACH(tbl, idx, entry) \ + for (idx = 0, (entry) = mlx5_l3t_get_next((tbl), &idx); \ + (entry); \ + idx++, (entry) = mlx5_l3t_get_next((tbl), &idx)) + #endif /* RTE_PMD_MLX5_UTILS_H_ */ From patchwork Wed Mar 31 07:36:20 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90182 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 74FB6A034F; Wed, 31 Mar 2021 09:38:05 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id E2113406FF; Wed, 31 Mar 2021 09:37:00 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 755C9140DED for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:40 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwA001778; Wed, 31 Mar 2021 10:36:40 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com, Suanming Mou Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com, Shun Hao , stable@dpdk.org Date: Wed, 31 Mar 2021 10:36:20 +0300 Message-Id: <20210331073632.1443011-3-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 02/13] net/mlx5: fix meter statistics X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Shun Hao This fixes the meter statistics issue that when using multiple meters, only one meter has stats value. To match the correct meter in policer table, now the meter_id is also used in its match criteria, so only one color and one drop matcher are needed. And both meter_id and flow_id will be written to related registers in meter prefix flow. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao --- drivers/net/mlx5/linux/mlx5_os.c | 9 +- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 25 +- drivers/net/mlx5/mlx5_flow.c | 182 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 7 files changed, 512 insertions(+), 293 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 2d5bcab4cf..b7eca9d153 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -316,7 +316,13 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) } sh->tx_domain = domain; #ifdef HAVE_MLX5DV_DR_ESWITCH + sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); if (priv->config.dv_esw_en) { + if (!sh->esw_drop_action) { + DRV_LOG(ERR, "Failed to create eswitch drop action"); + err = errno; + goto error; + } domain = mlx5_glue->dr_create_domain (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB); if (!domain) { @@ -325,7 +331,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) goto error; } sh->fdb_domain = domain; - sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); } #endif if (!sh->tunnel_hub) @@ -1273,7 +1278,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, - 1 + REG_C_0; priv->mtr_en = 1; priv->mtr_reg_share = - config->hca_attr.qos.flow_meter; + config->hca_attr.qos.flow_meter; DRV_LOG(DEBUG, "The REG_C meter uses is %d", priv->mtr_color_reg); } diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 3538cc8c20..d41c098f65 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 6faba4fbb1..95469c605a 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -188,6 +188,16 @@ struct mlx5_stats_ctrl { /* Maximal number of segments to split. */ #define MLX5_MAX_RXQ_NSEG (1u << MLX5_MAX_LOG_RQ_SEGS) +/* The bit size of one register. */ +#define MLX5_REG_BITS 32 + +/* Idle bits for non-color usage in color register. */ +#define MLX5_MTR_IDLE_BITS_IN_COLOR_REG (MLX5_REG_BITS - MLX5_MTR_COLOR_BITS) + +#define UINT32_T(x) ((uint32_t)(x)) + +#define LS32_MASK(bits) ((UINT32_T(1) << (bits)) - 1) + /* LRO configurations structure. */ struct mlx5_lro_config { uint32_t supported:1; /* Whether LRO is supported. */ @@ -944,9 +954,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1003,6 +1013,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1271,11 +1285,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index c347f8130e..cffd6129e8 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -764,9 +764,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3051,7 +3051,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3690,23 +3691,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4363,8 +4371,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4379,29 +4389,38 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint8_t flow_id_bits = 0; + int shift; + uint32_t flow_id_val = 0; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4410,10 +4429,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4446,23 +4462,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + /* Generate meter flow_id only if support multiple flows per meter. */ + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id_bits = MLX5_REG_BITS - __builtin_clz(tag_id - 1); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4491,16 +4506,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_val = (flow_id_val << 1) | + (((tag_id - 1) >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_val << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5200,25 +5233,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5232,9 +5289,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5245,10 +5302,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6520,20 +6579,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6558,7 +6614,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6571,14 +6627,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 8324e188e1..c92e746a04 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1148,6 +1153,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1188,8 +1194,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1252,7 +1257,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1452,11 +1457,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 23e5849783..ce9857d3f7 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10982,14 +10982,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11414,33 +11412,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12536,6 +12514,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12546,8 +12525,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12589,6 +12566,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13274,49 +13255,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13335,37 +13287,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13390,41 +13322,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13433,20 +13331,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13457,37 +13351,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13499,24 +13377,153 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | + LS32_MASK(MLX5_MTR_COLOR_BITS))); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + LS32_MASK(MLX5_MTR_COLOR_BITS)); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; + } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13537,7 +13544,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13546,39 +13553,55 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = LS32_MASK(priv->max_mtr_bits) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13591,25 +13614,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->esw_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13617,7 +13667,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13630,41 +13681,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initilized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -13960,7 +14077,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..68f5c344cf 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->esw_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** From patchwork Wed Mar 31 07:36:21 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90173 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 56B6FA034F; Wed, 31 Mar 2021 09:37:01 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 2A290140E02; Wed, 31 Mar 2021 09:36:49 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 7FF7D140DF1 for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:41 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwB001778; Wed, 31 Mar 2021 10:36:41 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:21 +0300 Message-Id: <20210331073632.1443011-4-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 03/13] net/mlx5: optimize meter statistics X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" 1. It does not support yellow counter and return 0. 2. All the meter colors with drop action will be counted only by the global drop statistics. 3. Red color must be with drop action. Signed-off-by: Li Zhang --- doc/guides/nics/mlx5.rst | 6 + drivers/net/mlx5/mlx5_flow.h | 24 ++-- drivers/net/mlx5/mlx5_flow_dv.c | 8 +- drivers/net/mlx5/mlx5_flow_meter.c | 217 +++++++++++++++++++---------- 4 files changed, 173 insertions(+), 82 deletions(-) diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst index b39bc475ad..e20ba3e6fd 100644 --- a/doc/guides/nics/mlx5.rst +++ b/doc/guides/nics/mlx5.rst @@ -398,6 +398,12 @@ Limitations - Hairpin between two ports could only manual binding and explicit Tx flow mode. For single port hairpin, all the combinations of auto/manual binding and explicit/implicit Tx flow mode could be supported. - Hairpin in switchdev SR-IOV mode is not supported till now. +- Meter: + - All the meter colors with drop action will be counted only by the global drop statistics. + - Green color is not supported with drop action. + - Yellow detection is not supported. + - Red color must be with drop action. + Statistics ---------- diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index c92e746a04..76870b8061 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -831,10 +831,10 @@ struct mlx5_flow { /* Meter policer statistics */ struct mlx5_flow_policer_stats { - uint32_t cnt[RTE_COLORS + 1]; - /**< Color counter, extra for drop. */ - uint64_t stats_mask; - /**< Statistics mask for the colors. */ + uint32_t pass_cnt; + /**< Color counter for pass. */ + uint32_t drop_cnt; + /**< Color counter for drop. */ }; /* Meter table structure. */ @@ -892,10 +892,18 @@ struct mlx5_flow_meter { /** Policer actions (per meter output color). */ enum rte_mtr_policer_action action[RTE_COLORS]; - /** Set of stats counters to be enabled. - * @see enum rte_mtr_stats_type - */ - uint64_t stats_mask; + uint32_t green_bytes:1; + /** Set green bytes stats to be enabled. */ + uint32_t green_pkts:1; + /** Set green packets stats to be enabled. */ + uint32_t red_bytes:1; + /** Set red bytes stats to be enabled. */ + uint32_t red_pkts:1; + /** Set red packets stats to be enabled. */ + uint32_t bytes_dropped:1; + /** Set bytes dropped stats to be enabled. */ + uint32_t pkts_dropped:1; + /** Set packets dropped stats to be enabled. */ /**< Rule applies to ingress traffic. */ uint32_t ingress:1; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index ce9857d3f7..d5c8f32038 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -13697,17 +13697,17 @@ flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, int ret; /* Get the statistics counters for green/drop. */ - if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + if (fm->policer_stats.pass_cnt) { cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[RTE_COLOR_GREEN], + fm->policer_stats.pass_cnt, NULL); mtb->green_count = cnt->action; } else { mtb->green_count = NULL; } - if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + if (fm->policer_stats.drop_cnt) { cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[RTE_MTR_DROPPED], + fm->policer_stats.drop_cnt, NULL); mtb->drop_count = cnt->action; } else { diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index 68f5c344cf..73b6e771d9 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -467,13 +467,6 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, struct rte_mtr_params *params, struct rte_mtr_error *error) { - static enum rte_mtr_policer_action - valid_recol_action[RTE_COLORS] = { - MTR_POLICER_ACTION_COLOR_GREEN, - MTR_POLICER_ACTION_COLOR_YELLOW, - MTR_POLICER_ACTION_COLOR_RED }; - int i; - /* Meter must use global drop action. */ if (!priv->sh->esw_drop_action) return -rte_mtr_error_set(error, ENOTSUP, @@ -493,13 +486,18 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, "Previous meter color " "not supported."); /* Validate policer settings. */ - for (i = 0; i < RTE_COLORS; i++) - if (params->action[i] != valid_recol_action[i] && - params->action[i] != MTR_POLICER_ACTION_DROP) - return -rte_mtr_error_set - (error, ENOTSUP, - action2error(params->action[i]), NULL, - "Recolor action not supported."); + if (params->action[RTE_COLOR_RED] != MTR_POLICER_ACTION_DROP) + return -rte_mtr_error_set + (error, ENOTSUP, + action2error(params->action[RTE_COLOR_RED]), + NULL, + "Red color only supports drop action."); + if (params->action[RTE_COLOR_GREEN] != MTR_POLICER_ACTION_COLOR_GREEN) + return -rte_mtr_error_set + (error, ENOTSUP, + action2error(params->action[RTE_COLOR_GREEN]), + NULL, + "Green color only supports recolor green action."); /* Validate meter id. */ if (mlx5_flow_meter_find(priv, meter_id)) return -rte_mtr_error_set(error, EEXIST, @@ -605,6 +603,19 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, #endif } +static void +mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter *fm, + uint64_t stats_mask) +{ + fm->green_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_GREEN) ? 1 : 0; + fm->green_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_GREEN) ? 1 : 0; + fm->red_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_RED) ? 1 : 0; + fm->red_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_RED) ? 1 : 0; + fm->bytes_dropped = + (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; + fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; +} + /** * Create meter rules. * @@ -643,7 +654,6 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .type = "mlx5_flow_mtr_flow_id_pool", }; int ret; - unsigned int i; uint32_t idx = 0; uint8_t mtr_id_bits; uint8_t mtr_reg_bits = priv->mtr_reg_share ? @@ -681,12 +691,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->meter_id = meter_id; fm->profile = fmp; memcpy(fm->action, params->action, sizeof(params->action)); - fm->stats_mask = params->stats_mask; + mlx5_flow_meter_stats_enable_update(fm, params->stats_mask); /* Alloc policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) { - fm->policer_stats.cnt[i] = mlx5_counter_alloc(dev); - if (!fm->policer_stats.cnt[i]) + if (fm->green_bytes || fm->green_pkts) { + fm->policer_stats.pass_cnt = mlx5_counter_alloc(dev); + if (!fm->policer_stats.pass_cnt) + goto error; + } + if (fm->red_bytes || fm->red_pkts || + fm->bytes_dropped || fm->pkts_dropped) { + fm->policer_stats.drop_cnt = mlx5_counter_alloc(dev); + if (!fm->policer_stats.drop_cnt) goto error; } fm->mfts = mlx5_flow_create_mtr_tbls(dev); @@ -699,7 +715,6 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, TAILQ_INSERT_TAIL(fms, fm, next); fm->active_state = 1; /* Config meter starts as active. */ fm->shared = !!shared; - fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); if (!fm->flow_ipool) @@ -710,9 +725,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); /* Free policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) - if (fm->policer_stats.cnt[i]) - mlx5_counter_free(dev, fm->policer_stats.cnt[i]); + if (fm->policer_stats.pass_cnt) + mlx5_counter_free(dev, fm->policer_stats.pass_cnt); + if (fm->policer_stats.drop_cnt) + mlx5_counter_free(dev, fm->policer_stats.drop_cnt); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], idx); return -rte_mtr_error_set(error, -ret, RTE_MTR_ERROR_TYPE_UNSPECIFIED, @@ -745,7 +761,6 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; - unsigned int i; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -770,9 +785,10 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, /* Remove from the flow meter list. */ TAILQ_REMOVE(fms, fm, next); /* Free policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) - if (fm->policer_stats.cnt[i]) - mlx5_counter_free(dev, fm->policer_stats.cnt[i]); + if (fm->policer_stats.pass_cnt) + mlx5_counter_free(dev, fm->policer_stats.pass_cnt); + if (fm->policer_stats.drop_cnt) + mlx5_counter_free(dev, fm->policer_stats.drop_cnt); /* Free meter flow table */ if (fm->flow_ipool) mlx5_ipool_destroy(fm->flow_ipool); @@ -1005,6 +1021,13 @@ mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_meter *fm; + const struct rte_flow_attr attr = { + .ingress = 1, + .egress = 1, + .transfer = priv->config.dv_esw_en ? 1 : 0, + }; + bool need_updated = false; + struct mlx5_flow_policer_stats old_policer_stats; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -1016,7 +1039,70 @@ mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, NULL, "Meter object id not valid."); - fm->policer_stats.stats_mask = stats_mask; + old_policer_stats.pass_cnt = 0; + old_policer_stats.drop_cnt = 0; + if (!!((RTE_MTR_STATS_N_PKTS_GREEN | + RTE_MTR_STATS_N_BYTES_GREEN) & stats_mask) != + !!fm->policer_stats.pass_cnt) { + need_updated = true; + if (fm->policer_stats.pass_cnt) { + old_policer_stats.pass_cnt = fm->policer_stats.pass_cnt; + fm->policer_stats.pass_cnt = 0; + } else { + fm->policer_stats.pass_cnt = + mlx5_counter_alloc(dev); + if (!fm->policer_stats.pass_cnt) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Counter alloc failed for meter."); + } + } + if (!!((RTE_MTR_STATS_N_PKTS_RED | RTE_MTR_STATS_N_BYTES_RED | + RTE_MTR_STATS_N_PKTS_DROPPED | RTE_MTR_STATS_N_BYTES_DROPPED) & + stats_mask) != + !!fm->policer_stats.drop_cnt) { + need_updated = true; + if (fm->policer_stats.drop_cnt) { + old_policer_stats.drop_cnt = fm->policer_stats.drop_cnt; + fm->policer_stats.drop_cnt = 0; + } else { + fm->policer_stats.drop_cnt = + mlx5_counter_alloc(dev); + if (!fm->policer_stats.drop_cnt) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Counter alloc failed for meter."); + } + } + if (need_updated) { + if (mlx5_flow_prepare_policer_rules(dev, fm, &attr)) { + if (fm->policer_stats.pass_cnt && + fm->policer_stats.pass_cnt != + old_policer_stats.pass_cnt) + mlx5_counter_free(dev, + fm->policer_stats.pass_cnt); + fm->policer_stats.pass_cnt = + old_policer_stats.pass_cnt; + if (fm->policer_stats.drop_cnt && + fm->policer_stats.drop_cnt != + old_policer_stats.drop_cnt) + mlx5_counter_free(dev, + fm->policer_stats.drop_cnt); + fm->policer_stats.pass_cnt = + old_policer_stats.pass_cnt; + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, + NULL, "Failed to create meter policer rules."); + } + /* Free old policer counters. */ + if (old_policer_stats.pass_cnt) + mlx5_counter_free(dev, + old_policer_stats.pass_cnt); + if (old_policer_stats.drop_cnt) + mlx5_counter_free(dev, + old_policer_stats.drop_cnt); + } + mlx5_flow_meter_stats_enable_update(fm, stats_mask); return 0; } @@ -1047,20 +1133,11 @@ mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, int clear, struct rte_mtr_error *error) { - static uint64_t meter2mask[RTE_MTR_DROPPED + 1] = { - RTE_MTR_STATS_N_PKTS_GREEN | RTE_MTR_STATS_N_BYTES_GREEN, - RTE_MTR_STATS_N_PKTS_YELLOW | RTE_MTR_STATS_N_BYTES_YELLOW, - RTE_MTR_STATS_N_PKTS_RED | RTE_MTR_STATS_N_BYTES_RED, - RTE_MTR_STATS_N_PKTS_DROPPED | RTE_MTR_STATS_N_BYTES_DROPPED - }; struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_meter *fm; struct mlx5_flow_policer_stats *ps; - uint64_t pkts_dropped = 0; - uint64_t bytes_dropped = 0; uint64_t pkts; uint64_t bytes; - int i; int ret = 0; if (!priv->mtr_en) @@ -1074,41 +1151,42 @@ mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_MTR_ID, NULL, "Meter object id not valid."); ps = &fm->policer_stats; - *stats_mask = ps->stats_mask; - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (*stats_mask & meter2mask[i]) { - ret = mlx5_counter_query(dev, ps->cnt[i], clear, &pkts, + *stats_mask = 0; + if (fm->green_bytes) + *stats_mask |= RTE_MTR_STATS_N_BYTES_GREEN; + if (fm->green_pkts) + *stats_mask |= RTE_MTR_STATS_N_PKTS_GREEN; + if (fm->red_bytes) + *stats_mask |= RTE_MTR_STATS_N_BYTES_RED; + if (fm->red_pkts) + *stats_mask |= RTE_MTR_STATS_N_PKTS_RED; + if (fm->bytes_dropped) + *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; + if (fm->pkts_dropped) + *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; + memset(stats, 0, sizeof(*stats)); + if (ps->pass_cnt) { + ret = mlx5_counter_query(dev, ps->pass_cnt, clear, &pkts, &bytes); - if (ret) - goto error; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) { - pkts_dropped += pkts; - bytes_dropped += bytes; - } - /* If need to read the packets, set it. */ - if ((1 << i) & (*stats_mask & meter2mask[i])) - stats->n_pkts[i] = pkts; - /* If need to read the bytes, set it. */ - if ((1 << (RTE_MTR_DROPPED + 1 + i)) & - (*stats_mask & meter2mask[i])) - stats->n_bytes[i] = bytes; - } + if (ret) + goto error; + /* If need to read the packets, set it. */ + if (fm->green_pkts) + stats->n_pkts[RTE_COLOR_GREEN] = pkts; + /* If need to read the bytes, set it. */ + if (fm->green_bytes) + stats->n_bytes[RTE_COLOR_GREEN] = bytes; } - /* Dropped packets/bytes are treated differently. */ - if (*stats_mask & meter2mask[i]) { - ret = mlx5_counter_query(dev, ps->cnt[i], clear, &pkts, - &bytes); + if (ps->drop_cnt) { + ret = mlx5_counter_query(dev, ps->drop_cnt, clear, &pkts, + &bytes); if (ret) goto error; - pkts += pkts_dropped; - bytes += bytes_dropped; /* If need to read the packets, set it. */ - if ((*stats_mask & meter2mask[i]) & - RTE_MTR_STATS_N_PKTS_DROPPED) + if (fm->pkts_dropped) stats->n_pkts_dropped = pkts; /* If need to read the bytes, set it. */ - if ((*stats_mask & meter2mask[i]) & - RTE_MTR_STATS_N_BYTES_DROPPED) + if (fm->bytes_dropped) stats->n_bytes_dropped = bytes; } return 0; @@ -1284,7 +1362,6 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) .transfer = priv->config.dv_esw_en ? 1 : 0, }; void *tmp; - uint32_t i; TAILQ_FOREACH_SAFE(fm, fms, next, tmp) { /* Meter object must not have any owner. */ @@ -1300,10 +1377,10 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) /* Remove from list. */ TAILQ_REMOVE(fms, fm, next); /* Free policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) - if (fm->policer_stats.cnt[i]) - mlx5_counter_free(dev, - fm->policer_stats.cnt[i]); + if (fm->policer_stats.pass_cnt) + mlx5_counter_free(dev, fm->policer_stats.pass_cnt); + if (fm->policer_stats.drop_cnt) + mlx5_counter_free(dev, fm->policer_stats.drop_cnt); /* Free meter flow table. */ mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); From patchwork Wed Mar 31 07:36:22 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90172 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6BDACA034F; Wed, 31 Mar 2021 09:36:55 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id E7747140DF1; Wed, 31 Mar 2021 09:36:47 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 8614A140DF9 for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:41 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwC001778; Wed, 31 Mar 2021 10:36:41 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com, Shun Hao Date: Wed, 31 Mar 2021 10:36:22 +0300 Message-Id: <20210331073632.1443011-5-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 04/13] net/mlx5: use mask for meter register setting X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Shun Hao ASO meter feature may require to locate the flow context tag action after the ASO action. When color register is shared by meter_id/flow_id, it's like: Bits[0-7] A meter color value set by the HW. Bits[8-31] A flow id and meter id set by SW. Currently the tag action for meter writes all the bits of the meter register, so it will potentially overwrite meter color when ASO meter action is before the tag action. Set only 24-MSB-bits of meter register in the meter tag action. Signed-off-by: Shun Hao --- drivers/net/mlx5/mlx5_flow.c | 27 +++++++++++++++++---------- drivers/net/mlx5/mlx5_flow.h | 2 ++ drivers/net/mlx5/mlx5_flow_dv.c | 2 ++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index cffd6129e8..a4bed659f2 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -4274,9 +4274,11 @@ flow_hairpin_split(struct rte_eth_dev *dev, rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); actions_rx++; set_tag = (void *)actions_rx; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), + .data = flow_id, + }; MLX5_ASSERT(set_tag->id > REG_NON); - set_tag->data = flow_id; tag_action->conf = set_tag; /* Create Tx item list. */ rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); @@ -4511,6 +4513,13 @@ flow_meter_split_prep(struct rte_eth_dev *dev, set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; tag_item_mask = tag_item_spec + 1; + /* Both flow_id and meter_id share the same register. */ + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error), + .offset = mtr_id_offset, + .length = mtr_reg_bits, + .data = fm->idx, + }; /* * The color Reg bits used by flow_id are growing from * msb to lsb, so must do bit reverse for flow_id val in RegC. @@ -4518,12 +4527,8 @@ flow_meter_split_prep(struct rte_eth_dev *dev, for (shift = 0; shift < flow_id_bits; shift++) flow_id_val = (flow_id_val << 1) | (((tag_id - 1) >> shift) & 0x1); - /* Both flow_id and meter_id share the same register. */ - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); - set_tag->data = - (fm->idx | (flow_id_val << (mtr_reg_bits - flow_id_bits))) - << mtr_id_offset; - tag_item_spec->id = set_tag->id; + set_tag->data |= flow_id_val << (mtr_reg_bits - flow_id_bits); + tag_item_spec->id = set_tag->id << mtr_id_offset; tag_item_spec->data = set_tag->data; tag_item_mask->data = UINT32_MAX << mtr_id_offset; tag_action->type = (enum rte_flow_action_type) @@ -4911,10 +4916,12 @@ flow_sample_split_prep(struct rte_eth_dev *dev, ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); if (ret < 0) return ret; - set_tag->id = ret; mlx5_ipool_malloc(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); - set_tag->data = tag_id; + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = ret, + .data = tag_id, + }; /* Prepare the suffix subflow items. */ tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); tag_spec->data = tag_id; diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 76870b8061..e7f0906209 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -55,6 +55,8 @@ struct mlx5_rte_flow_item_tag { /* Modify selected register. */ struct mlx5_rte_flow_action_set_tag { enum modify_reg id; + uint8_t offset; + uint8_t length; uint32_t data; }; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index d5c8f32038..30fec09987 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -964,6 +964,8 @@ flow_dv_convert_action_set_reg actions[i] = (struct mlx5_modification_cmd) { .action_type = MLX5_MODIFICATION_TYPE_SET, .field = reg_to_field[conf->id], + .offset = conf->offset, + .length = conf->length, }; actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); actions[i].data1 = rte_cpu_to_be_32(conf->data); From patchwork Wed Mar 31 07:36:23 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90174 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id D6C77A034F; Wed, 31 Mar 2021 09:37:08 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 677CC140E0C; Wed, 31 Mar 2021 09:36:50 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 8A0CE140DFB for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:41 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwD001778; Wed, 31 Mar 2021 10:36:41 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:23 +0300 Message-Id: <20210331073632.1443011-6-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 05/13] common/mlx5: add definitions for ASO flow meter X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch adds different PRM definitions, related to ASO flow meter feature, in MLX5 PMD code. Signed-off-by: Li Zhang --- drivers/common/mlx5/mlx5_prm.h | 75 ++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h index 0ef0574f92..6d6c7f38bb 100644 --- a/drivers/common/mlx5/mlx5_prm.h +++ b/drivers/common/mlx5/mlx5_prm.h @@ -1129,6 +1129,8 @@ enum { (1ULL << MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH) #define MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO \ (1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO) +#define MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO \ + (1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO) #define MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT \ (1ULL << MLX5_OBJ_TYPE_GENEVE_TLV_OPT) @@ -1514,7 +1516,15 @@ struct mlx5_ifc_qos_cap_bits { u8 reserved_at_c0[0x10]; u8 max_qos_para_vport[0x10]; u8 max_tsar_bw_share[0x20]; - u8 reserved_at_100[0x6e8]; + u8 nic_element_type[0x10]; + u8 nic_tsar_type[0x10]; + u8 reserved_at_120[0x3]; + u8 log_meter_aso_granularity[0x5]; + u8 reserved_at_128[0x3]; + u8 log_meter_aso_max_alloc[0x5]; + u8 reserved_at_130[0x3]; + u8 log_max_num_meter_aso[0x5]; + u8 reserved_at_138[0x6b0]; }; struct mlx5_ifc_per_protocol_networking_offload_caps_bits { @@ -2284,6 +2294,8 @@ struct mlx5_ifc_flow_meter_parameters_bits { u8 eir_mantissa[0x8]; u8 reserved_at_8[0x60]; // 14h-1Ch }; +#define MLX5_IFC_FLOW_METER_PARAM_MASK UINT64_C(0x80FFFFFF) +#define MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL 0x14BF00C8 enum { MLX5_CQE_SIZE_64B = 0x0, @@ -2411,6 +2423,7 @@ enum { MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d, MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c, MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022, + MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO = 0x0024, MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025, }; @@ -2419,7 +2432,9 @@ struct mlx5_ifc_general_obj_in_cmd_hdr_bits { u8 reserved_at_10[0x20]; u8 obj_type[0x10]; u8 obj_id[0x20]; - u8 reserved_at_60[0x20]; + u8 reserved_at_60[0x3]; + u8 log_obj_range[0x5]; + u8 reserved_at_58[0x18]; }; struct mlx5_ifc_general_obj_out_cmd_hdr_bits { @@ -2565,6 +2580,18 @@ struct mlx5_ifc_create_flow_hit_aso_in_bits { struct mlx5_ifc_flow_hit_aso_bits flow_hit_aso; }; +struct mlx5_ifc_flow_meter_aso_bits { + u8 modify_field_select[0x40]; + u8 reserved_at_40[0x48]; + u8 access_pd[0x18]; + u8 reserved_at_a0[0x160]; + u8 parameters[0x200]; +}; + +struct mlx5_ifc_create_flow_meter_aso_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr; + struct mlx5_ifc_flow_meter_aso_bits flow_meter_aso; +}; enum mlx5_access_aso_opc_mod { ASO_OPC_MOD_IPSEC = 0x0, ASO_OPC_MOD_CONNECTION_TRACKING = 0x1, @@ -2618,11 +2645,51 @@ struct mlx5_aso_cseg { uint64_t data_mask; } __rte_packed; +/* A meter data segment - 2 per ASO WQE. */ +struct mlx5_aso_mtr_dseg { + uint32_t v_bo_sc_bbog_mm; + /* + * bit 31: valid, 30: bucket overflow, 28-29: start color, + * 27: both buckets on green, 24-25: meter mode. + */ + uint32_t reserved; + uint32_t cbs_cir; + /* + * bit 24-28: cbs_exponent, bit 16-23 cbs_mantissa, + * bit 8-12: cir_exponent, bit 0-7 cir_mantissa. + */ + uint32_t c_tokens; + uint32_t ebs_eir; + /* + * bit 24-28: ebs_exponent, bit 16-23 ebs_mantissa, + * bit 8-12: eir_exponent, bit 0-7 eir_mantissa. + */ + uint32_t e_tokens; + uint64_t timestamp; +} __rte_packed; + +#define ASO_DSEG_VALID_OFFSET 31 +#define ASO_DSEG_BO_OFFSET 30 +#define ASO_DSEG_SC_OFFSET 28 +#define ASO_DSEG_CBS_EXP_OFFSET 24 +#define ASO_DSEG_CBS_MAN_OFFSET 16 +#define ASO_DSEG_CIR_EXP_MASK 0x1F +#define ASO_DSEG_CIR_EXP_OFFSET 8 +#define ASO_DSEG_EBS_EXP_OFFSET 24 +#define ASO_DSEG_EBS_MAN_OFFSET 16 +#define ASO_DSEG_EXP_MASK 0x1F +#define ASO_DSEG_MAN_MASK 0xFF + #define MLX5_ASO_WQE_DSEG_SIZE 0x40 +#define MLX5_ASO_METERS_PER_WQE 2 +#define MLX5_ASO_MTRS_PER_POOL 128 -/* ASO WQE Data segment. */ +/* ASO WQE data segment. */ struct mlx5_aso_dseg { - uint8_t data[MLX5_ASO_WQE_DSEG_SIZE]; + union { + uint8_t data[MLX5_ASO_WQE_DSEG_SIZE]; + struct mlx5_aso_mtr_dseg mtrs[MLX5_ASO_METERS_PER_WQE]; + }; } __rte_packed; /* ASO WQE. */ From patchwork Wed Mar 31 07:36:24 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90175 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 83192A034F; Wed, 31 Mar 2021 09:37:15 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id BFCEA140E13; Wed, 31 Mar 2021 09:36:51 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 8F41F140DFC for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:41 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwE001778; Wed, 31 Mar 2021 10:36:41 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:24 +0300 Message-Id: <20210331073632.1443011-7-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 06/13] common/mlx5: add read ASO flow meter HCA capability X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Read and store the device capability of FLOW_METER_ASO general object, using the DevX API. Signed-off-by: Li Zhang --- drivers/common/mlx5/mlx5_devx_cmds.c | 14 ++++++++++++++ drivers/common/mlx5/mlx5_devx_cmds.h | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index c90e020643..aaeb49b6ba 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -695,6 +695,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, attr->max_geneve_tlv_option_data_len = MLX5_GET(cmd_hca_cap, hcattr, max_geneve_tlv_option_data_len); attr->qos.sup = MLX5_GET(cmd_hca_cap, hcattr, qos); + attr->qos.flow_meter_aso_sup = !!(MLX5_GET64(cmd_hca_cap, hcattr, + general_obj_types) & + MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO); attr->vdpa.valid = !!(MLX5_GET64(cmd_hca_cap, hcattr, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q); @@ -778,6 +781,17 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, MLX5_GET(qos_cap, hcattr, packet_pacing); attr->qos.wqe_rate_pp = MLX5_GET(qos_cap, hcattr, wqe_rate_pp); + if (attr->qos.flow_meter_aso_sup) { + attr->qos.log_meter_aso_granularity = + MLX5_GET(qos_cap, hcattr, + log_meter_aso_granularity); + attr->qos.log_meter_aso_max_alloc = + MLX5_GET(qos_cap, hcattr, + log_meter_aso_max_alloc); + attr->qos.log_max_num_meter_aso = + MLX5_GET(qos_cap, hcattr, + log_max_num_meter_aso); + } } if (attr->vdpa.valid) mlx5_devx_cmd_query_hca_vdpa_attr(ctx, &attr->vdpa); diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h index 2826c0b2c6..8c68dfa5e0 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.h +++ b/drivers/common/mlx5/mlx5_devx_cmds.h @@ -49,10 +49,18 @@ struct mlx5_hca_qos_attr { * Using older driver versions, flow_meter_old can be 1 * while flow_meter is 0. */ + uint32_t flow_meter_aso_sup:1; + /* Whether FLOW_METER_ASO Object is supported. */ uint8_t log_max_flow_meter; /* Power of the maximum supported meters. */ uint8_t flow_meter_reg_c_ids; /* Bitmap of the reg_Cs available for flow meter to use. */ + uint32_t log_meter_aso_granularity:5; + /* Power of the minimum allocation granularity Object. */ + uint32_t log_meter_aso_max_alloc:5; + /* Power of the maximum allocation granularity Object. */ + uint32_t log_max_num_meter_aso:5; + /* Power of the maximum number of supported objects. */ }; From patchwork Wed Mar 31 07:36:25 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90176 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1C72BA034F; Wed, 31 Mar 2021 09:37:23 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 335B5140E22; Wed, 31 Mar 2021 09:36:53 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 9628B140DFD for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:42 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwF001778; Wed, 31 Mar 2021 10:36:41 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com, Ray Kinsella , Neil Horman Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:25 +0300 Message-Id: <20210331073632.1443011-8-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 07/13] common/mlx5: add DevX API to create ASO flow meter object X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add DevX API to create ASO flow meter object. Signed-off-by: Li Zhang --- drivers/common/mlx5/mlx5_devx_cmds.c | 54 ++++++++++++++++++++++++++++ drivers/common/mlx5/mlx5_devx_cmds.h | 18 +++++++++- drivers/common/mlx5/version.map | 1 + 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index aaeb49b6ba..bec5836392 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -2154,6 +2154,60 @@ mlx5_devx_cmd_alloc_pd(void *ctx) return ppd; } +/** + * Create general object of type FLOW_METER_ASO using DevX API. + * + * @param[in] ctx + * Context returned from mlx5 open_device() glue function. + * @param [in] pd + * PD value to associate the FLOW_METER_ASO object with. + * @param [in] log_obj_size + * log_obj_size define to allocate number of 2 * meters + * in one FLOW_METER_ASO object. + * + * @return + * The DevX object created, NULL otherwise and rte_errno is set. + */ +struct mlx5_devx_obj * +mlx5_devx_cmd_create_flow_meter_aso_obj(void *ctx, uint32_t pd, + uint32_t log_obj_size) +{ + uint32_t in[MLX5_ST_SZ_DW(create_flow_meter_aso_in)] = {0}; + uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + struct mlx5_devx_obj *flow_meter_aso_obj; + void *ptr; + + flow_meter_aso_obj = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*flow_meter_aso_obj), + 0, SOCKET_ID_ANY); + if (!flow_meter_aso_obj) { + DRV_LOG(ERR, "Failed to allocate FLOW_METER_ASO object data"); + rte_errno = ENOMEM; + return NULL; + } + ptr = MLX5_ADDR_OF(create_flow_meter_aso_in, in, hdr); + MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode, + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type, + MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO); + MLX5_SET(general_obj_in_cmd_hdr, ptr, log_obj_range, + log_obj_size); + ptr = MLX5_ADDR_OF(create_flow_meter_aso_in, in, flow_meter_aso); + MLX5_SET(flow_meter_aso, ptr, access_pd, pd); + flow_meter_aso_obj->obj = mlx5_glue->devx_obj_create( + ctx, in, sizeof(in), + out, sizeof(out)); + if (!flow_meter_aso_obj->obj) { + rte_errno = errno; + DRV_LOG(ERR, "Failed to create FLOW_METER_ASO obj using DevX."); + mlx5_free(flow_meter_aso_obj); + return NULL; + } + flow_meter_aso_obj->id = MLX5_GET(general_obj_out_cmd_hdr, + out, obj_id); + return flow_meter_aso_obj; +} + /** * Create general object of type GENEVE TLV option using DevX API. * diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h index 8c68dfa5e0..e399730c0e 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.h +++ b/drivers/common/mlx5/mlx5_devx_cmds.h @@ -551,7 +551,6 @@ int mlx5_devx_cmd_query_virtio_q_counters(struct mlx5_devx_obj *couners_obj, __rte_internal struct mlx5_devx_obj *mlx5_devx_cmd_create_flow_hit_aso_obj(void *ctx, uint32_t pd); - __rte_internal struct mlx5_devx_obj *mlx5_devx_cmd_alloc_pd(void *ctx); @@ -563,4 +562,21 @@ struct mlx5_devx_obj *mlx5_devx_cmd_queue_counter_alloc(void *ctx); __rte_internal int mlx5_devx_cmd_queue_counter_query(struct mlx5_devx_obj *dcs, int clear, uint32_t *out_of_buffers); +/** + * Create general object of type FLOW_METER_ASO using DevX API.. + * + * @param[in] ctx + * Device context. + * @param [in] pd + * PD value to associate the FLOW_METER_ASO object with. + * @param [in] log_obj_size + * log_obj_size define to allocate number of 2 * meters + * in one FLOW_METER_ASO object. + * + * @return + * The DevX object created, NULL otherwise and rte_errno is set. + */ +__rte_internal +struct mlx5_devx_obj *mlx5_devx_cmd_create_flow_meter_aso_obj(void *ctx, + uint32_t pd, uint32_t log_obj_size); #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */ diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map index 91f3fa5779..0b3a8a3db2 100644 --- a/drivers/common/mlx5/version.map +++ b/drivers/common/mlx5/version.map @@ -23,6 +23,7 @@ INTERNAL { mlx5_devx_cmd_create_virtio_q_counters; mlx5_devx_cmd_create_virtq; mlx5_devx_cmd_create_flow_hit_aso_obj; + mlx5_devx_cmd_create_flow_meter_aso_obj; mlx5_devx_cmd_create_geneve_tlv_option; mlx5_devx_cmd_destroy; mlx5_devx_cmd_flow_counter_alloc; From patchwork Wed Mar 31 07:36:26 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90181 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 963F7A034F; Wed, 31 Mar 2021 09:37:57 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 86E33140E4E; Wed, 31 Mar 2021 09:36:59 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 9C432140E00 for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:42 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwG001778; Wed, 31 Mar 2021 10:36:42 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:26 +0300 Message-Id: <20210331073632.1443011-9-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 08/13] net/mlx5: flow meter pool to manage meter object X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add ASO flow meter pool to manage meter object Signed-off-by: Li Zhang --- drivers/net/mlx5/mlx5.c | 2 +- drivers/net/mlx5/mlx5.h | 207 ++++++++++++- drivers/net/mlx5/mlx5_flow.c | 70 ++++- drivers/net/mlx5/mlx5_flow.h | 196 +++---------- drivers/net/mlx5/mlx5_flow_dv.c | 202 ++++++++++++- drivers/net/mlx5/mlx5_flow_meter.c | 450 ++++++++++++++++++----------- 6 files changed, 768 insertions(+), 359 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index d41c098f65..8e0582826d 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -280,7 +280,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { * for meter idx, so not set grow_trunk to avoid meter index * not jump continually. */ - .size = sizeof(struct mlx5_flow_meter), + .size = sizeof(struct mlx5_legacy_flow_meter), .trunk_size = 64, .need_lock = 1, .release_mem_en = 1, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 95469c605a..ced14dd0bf 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -583,6 +584,193 @@ struct mlx5_dev_shared_port { /* Aging information for per port. */ }; +/*ASO flow meter structures*/ +/* Modify this value if enum rte_mtr_color changes. */ +#define RTE_MTR_DROPPED RTE_COLORS + +/* Meter policer statistics */ +struct mlx5_flow_policer_stats { + uint32_t pass_cnt; + /**< Color counter for pass. */ + uint32_t drop_cnt; + /**< Color counter for drop. */ +}; + +/* Meter table structure. */ +struct mlx5_meter_domain_info { + struct mlx5_flow_tbl_resource *tbl; + /**< Meter table. */ + struct mlx5_flow_tbl_resource *sfx_tbl; + /**< Meter suffix table. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ + void *jump_actn; + /**< Meter match action. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ +}; + +/* Meter table set for TX RX FDB. */ +struct mlx5_meter_domains_infos { + uint32_t ref_cnt; + /**< Table user count. */ + struct mlx5_meter_domain_info egress; + /**< TX meter table. */ + struct mlx5_meter_domain_info ingress; + /**< RX meter table. */ + struct mlx5_meter_domain_info transfer; + /**< FDB meter table. */ + void *drop_actn; + /**< Drop action as not matched. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ + void *meter_action; + /**< Flow meter action. */ +}; + +/* Meter parameter structure. */ +struct mlx5_flow_meter_info { + uint32_t meter_id; + /**< Meter id. */ + struct mlx5_flow_meter_profile *profile; + /**< Meter profile parameters. */ + rte_spinlock_t sl; /**< Meter action spinlock. */ + /** Policer actions (per meter output color). */ + enum rte_mtr_policer_action action[RTE_COLORS]; + /** Set of stats counters to be enabled. + * @see enum rte_mtr_stats_type + */ + uint32_t green_bytes:1; + /** Set green bytes stats to be enabled. */ + uint32_t green_pkts:1; + /** Set green packets stats to be enabled. */ + uint32_t red_bytes:1; + /** Set red bytes stats to be enabled. */ + uint32_t red_pkts:1; + /** Set red packets stats to be enabled. */ + uint32_t bytes_dropped:1; + /** Set bytes dropped stats to be enabled. */ + uint32_t pkts_dropped:1; + /** Set packets dropped stats to be enabled. */ + uint32_t active_state:1; + /**< Meter hw active state. */ + uint32_t shared:1; + /**< Meter shared or not. */ + uint32_t is_enable:1; + /**< Meter disable/enable state. */ + uint32_t ingress:1; + /**< Rule applies to egress traffic. */ + uint32_t egress:1; + /** + * Instead of simply matching the properties of traffic as it would + * appear on a given DPDK port ID, enabling this attribute transfers + * a flow rule to the lowest possible level of any device endpoints + * found in the pattern. + * + * When supported, this effectively enables an application to + * re-route traffic not necessarily intended for it (e.g. coming + * from or addressed to different physical ports, VFs or + * applications) at the device level. + * + * It complements the behavior of some pattern items such as + * RTE_FLOW_ITEM_TYPE_PHY_PORT and is meaningless without them. + * + * When transferring flow rules, ingress and egress attributes keep + * their original meaning, as if processing traffic emitted or + * received by the application. + */ + uint32_t transfer:1; + struct mlx5_meter_domains_infos *mfts; + /**< Flow table created for this meter. */ + struct mlx5_flow_policer_stats policer_stats; + /**< Meter policer statistics. */ + uint32_t ref_cnt; + /**< Use count. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ +}; + +/* RFC2697 parameter structure. */ +struct mlx5_flow_meter_srtcm_rfc2697_prm { + rte_be32_t cbs_cir; + /* + * bit 24-28: cbs_exponent, bit 16-23 cbs_mantissa, + * bit 8-12: cir_exponent, bit 0-7 cir_mantissa. + */ + rte_be32_t ebs_eir; + /* + * bit 24-28: ebs_exponent, bit 16-23 ebs_mantissa, + * bit 8-12: eir_exponent, bit 0-7 eir_mantissa. + */ +}; + +/* Flow meter profile structure. */ +struct mlx5_flow_meter_profile { + TAILQ_ENTRY(mlx5_flow_meter_profile) next; + /**< Pointer to the next flow meter structure. */ + uint32_t id; /**< Profile id. */ + struct rte_mtr_meter_profile profile; /**< Profile detail. */ + union { + struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm_prm; + /**< srtcm_rfc2697 struct. */ + }; + uint32_t ref_cnt; /**< Use count. */ +}; + +/* 2 meters in each ASO cache line */ +#define MLX5_MTRS_CONTAINER_RESIZE 64 +/* + * The pool index and offset of meter in the pool array makes up the + * meter index. In case the meter is from pool 0 and offset 0, it + * should plus 1 to avoid index 0, since 0 means invalid meter index + * currently. + */ +#define MLX5_MAKE_MTR_IDX(pi, offset) \ + ((pi) * MLX5_ASO_MTRS_PER_POOL + (offset) + 1) + +/*aso flow meter state*/ +enum mlx5_aso_mtr_state { + ASO_METER_FREE, /* In free list. */ + ASO_METER_WAIT, /* ACCESS_ASO WQE in progress. */ + ASO_METER_READY, /* CQE received. */ +}; + +/* Generic aso_flow_meter information. */ +struct mlx5_aso_mtr { + LIST_ENTRY(mlx5_aso_mtr) next; + struct mlx5_flow_meter_info fm; + /**< Pointer to the next aso flow meter structure. */ + uint8_t state; /**< ASO flow meter state. */ + uint8_t offset; +}; + +/* Generic aso_flow_meter pool structure. */ +struct mlx5_aso_mtr_pool { + struct mlx5_aso_mtr mtrs[MLX5_ASO_MTRS_PER_POOL]; + /*Must be the first in pool*/ + struct mlx5_devx_obj *devx_obj; + /* The devx object of the minimum aso flow meter ID. */ + uint32_t index; /* Pool index in management structure. */ +}; + +LIST_HEAD(aso_meter_list, mlx5_aso_mtr); +/* Pools management structure for ASO flow meter pools. */ +struct mlx5_aso_mtr_pools_mng { + volatile uint16_t n_valid; /* Number of valid pools. */ + uint16_t n; /* Number of pools. */ + rte_spinlock_t mtrsl; /* The ASO flow meter free list lock. */ + struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */ + struct aso_meter_list meters; /* Free ASO flow meter list. */ + struct mlx5_aso_sq sq; /*SQ using by ASO flow meter. */ + struct mlx5_aso_mtr_pool **pools; /* ASO flow meter pool array. */ +}; + /* Table key of the hash organization. */ union mlx5_flow_tbl_key { struct { @@ -709,6 +897,7 @@ struct mlx5_dev_ctx_shared { uint32_t rq_ts_format:2; /* RQ timestamp formats supported. */ uint32_t sq_ts_format:2; /* SQ timestamp formats supported. */ uint32_t qp_ts_format:2; /* QP timestamp formats supported. */ + uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */ uint32_t max_port; /* Maximal IB device port index. */ struct mlx5_bond_info bond; /* Bonding information. */ void *ctx; /* Verbs/DV/DevX context. */ @@ -769,6 +958,8 @@ struct mlx5_dev_ctx_shared { struct mlx5_geneve_tlv_option_resource *geneve_tlv_option_resource; /* Management structure for geneve tlv option */ rte_spinlock_t geneve_tlv_opt_sl; /* Lock for geneve tlv resource */ + struct mlx5_aso_mtr_pools_mng *mtrmng; + /* Meter pools management structure. */ struct mlx5_dev_shared_port port[]; /* per device port data array. */ }; @@ -786,7 +977,7 @@ struct mlx5_proc_priv { /* MTR profile list. */ TAILQ_HEAD(mlx5_mtr_profiles, mlx5_flow_meter_profile); /* MTR list. */ -TAILQ_HEAD(mlx5_flow_meters, mlx5_flow_meter); +TAILQ_HEAD(mlx5_legacy_flow_meters, mlx5_legacy_flow_meter); /* RSS description. */ struct mlx5_flow_rss_desc { @@ -1004,7 +1195,7 @@ struct mlx5_priv { uint8_t mtr_sfx_reg; /* Meter prefix-suffix flow match REG_C. */ uint8_t mtr_color_reg; /* Meter color match REG_C. */ struct mlx5_mtr_profiles flow_meter_profiles; /* MTR profile list. */ - struct mlx5_flow_meters flow_meters; /* MTR list. */ + struct mlx5_legacy_flow_meters flow_meters; /* MTR list. */ uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */ uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */ struct mlx5_mp_id mp_id; /* ID of a multi-process process */ @@ -1283,13 +1474,15 @@ int mlx5_pmd_socket_init(void); /* mlx5_flow_meter.c */ int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); -struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, - uint32_t meter_id); +struct mlx5_flow_meter_info *mlx5_flow_meter_find(struct mlx5_priv *priv, + uint32_t meter_id, uint32_t *mtr_idx); +struct mlx5_flow_meter_info * +flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx); int mlx5_flow_meter_attach(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error); -void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); +void mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm); /* mlx5_os.c */ struct rte_pci_driver; @@ -1334,7 +1527,7 @@ void mlx5_txpp_interrupt_handler(void *cb_arg); eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev); -/* mlx5_flow_age.c */ +/* mlx5_flow_aso.c */ int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh); int mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh); diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index a4bed659f2..3fe37b92ee 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -4375,6 +4375,8 @@ flow_create_split_inner(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. * @param[in] fm * Pointer to flow meter structure. * @param[in] items @@ -4387,10 +4389,6 @@ flow_create_split_inner(struct rte_eth_dev *dev, * Suffix flow actions. * @param[out] actions_pre * Prefix flow actions. - * @param[out] pattern_sfx - * The pattern items for the suffix flow. - * @param[out] tag_sfx - * Pointer to suffix flow tag. * @param[out] error * Perform verbose error reporting if not NULL. * @@ -4399,7 +4397,8 @@ flow_create_split_inner(struct rte_eth_dev *dev, */ static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct rte_flow *flow, + struct mlx5_flow_meter_info *fm, const struct rte_flow_item items[], struct rte_flow_item sfx_items[], const struct rte_flow_action actions[], @@ -4518,7 +4517,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, .id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error), .offset = mtr_id_offset, .length = mtr_reg_bits, - .data = fm->idx, + .data = flow->meter, }; /* * The color Reg bits used by flow_id are growing from @@ -5246,9 +5245,10 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - struct mlx5_flow_meter *fm = NULL; + struct mlx5_flow_meter_info *fm = NULL; bool has_mtr = false; uint32_t meter_id; + uint32_t mtr_idx = 0; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; @@ -5260,14 +5260,13 @@ flow_create_split_meter(struct rte_eth_dev *dev, &meter_id); if (has_mtr) { if (flow->meter) { - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], - flow->meter); + fm = flow_dv_meter_find_by_idx(priv, flow->meter); if (!fm) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "Meter not found."); } else { - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); if (!fm) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -5276,7 +5275,7 @@ flow_create_split_meter(struct rte_eth_dev *dev, &sfx_attr, error); if (ret) return -rte_errno; - flow->meter = fm->idx; + flow->meter = mtr_idx; } wks->fm = fm; /* The prefix actions: meter, decap, encap, tag, end. */ @@ -5296,9 +5295,10 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, - actions, sfx_actions, - pre_actions, error); + mtr_tag_id = flow_meter_split_prep(dev, flow, fm, items, + sfx_items, actions, + sfx_actions, pre_actions, + error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -6635,7 +6635,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, */ int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; @@ -6657,7 +6657,7 @@ mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, */ int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; @@ -6666,6 +6666,44 @@ mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, return fops->destroy_policer_rules(dev, fm, attr); } +/** + * Allocate the needed aso flow meter id. + * + * @param[in] dev + * Pointer to Ethernet device. + * + * @return + * Index to aso flow meter on success, NULL otherwise. + */ +uint32_t +mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_meter(dev); +} + +/** + * Free the aso flow meter id. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] mtr_idx + * Index to aso flow meter to be free. + * + * @return + * 0 on success. + */ +void +mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->free_meter(dev, mtr_idx); +} + /** * Allocate a counter. * diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index e7f0906209..8debe1c845 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -827,148 +827,17 @@ struct mlx5_flow { #define MLX5_FLOW_METER_DISABLE 0 #define MLX5_FLOW_METER_ENABLE 1 -#define MLX5_MAN_WIDTH 8 -/* Modify this value if enum rte_mtr_color changes. */ -#define RTE_MTR_DROPPED RTE_COLORS - -/* Meter policer statistics */ -struct mlx5_flow_policer_stats { - uint32_t pass_cnt; - /**< Color counter for pass. */ - uint32_t drop_cnt; - /**< Color counter for drop. */ -}; - -/* Meter table structure. */ -struct mlx5_meter_domain_info { - struct mlx5_flow_tbl_resource *tbl; - /**< Meter table. */ - struct mlx5_flow_tbl_resource *sfx_tbl; - /**< Meter suffix table. */ - struct mlx5_flow_dv_matcher *drop_matcher; - /**< Matcher for Drop. */ - struct mlx5_flow_dv_matcher *color_matcher; - /**< Matcher for Color. */ - void *jump_actn; - /**< Meter match action. */ - void *green_rule; - /**< Meter green rule. */ - void *drop_rule; - /**< Meter drop rule. */ -}; - -/* Meter table set for TX RX FDB. */ -struct mlx5_meter_domains_infos { - uint32_t ref_cnt; - /**< Table user count. */ - struct mlx5_meter_domain_info egress; - /**< TX meter table. */ - struct mlx5_meter_domain_info ingress; - /**< RX meter table. */ - struct mlx5_meter_domain_info transfer; - /**< FDB meter table. */ - void *green_count; - /**< Counters for green rule. */ - void *drop_count; - /**< Counters for green rule. */ - uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; - /**< Flow meter parameter. */ - size_t fmp_size; - /**< Flow meter parameter size. */ - void *meter_action; - /**< Flow meter action. */ -}; +#define MLX5_ASO_CQE_RESPONSE_DELAY 10 +#define MLX5_MTR_POLL_CQE_TIMES 100000u -/* Meter parameter structure. */ -struct mlx5_flow_meter { - TAILQ_ENTRY(mlx5_flow_meter) next; +#define MLX5_MAN_WIDTH 8 +/* Legacy Meter parameter structure. */ +struct mlx5_legacy_flow_meter { + struct mlx5_flow_meter_info fm; + /* Must be the first in struct. */ + TAILQ_ENTRY(mlx5_legacy_flow_meter) next; /**< Pointer to the next flow meter structure. */ uint32_t idx; /* Index to meter object. */ - uint32_t meter_id; - /**< Meter id. */ - struct mlx5_flow_meter_profile *profile; - /**< Meter profile parameters. */ - - rte_spinlock_t sl; /**< Meter action spinlock. */ - - /** Policer actions (per meter output color). */ - enum rte_mtr_policer_action action[RTE_COLORS]; - - uint32_t green_bytes:1; - /** Set green bytes stats to be enabled. */ - uint32_t green_pkts:1; - /** Set green packets stats to be enabled. */ - uint32_t red_bytes:1; - /** Set red bytes stats to be enabled. */ - uint32_t red_pkts:1; - /** Set red packets stats to be enabled. */ - uint32_t bytes_dropped:1; - /** Set bytes dropped stats to be enabled. */ - uint32_t pkts_dropped:1; - /** Set packets dropped stats to be enabled. */ - - /**< Rule applies to ingress traffic. */ - uint32_t ingress:1; - - /**< Rule applies to egress traffic. */ - uint32_t egress:1; - /** - * Instead of simply matching the properties of traffic as it would - * appear on a given DPDK port ID, enabling this attribute transfers - * a flow rule to the lowest possible level of any device endpoints - * found in the pattern. - * - * When supported, this effectively enables an application to - * re-route traffic not necessarily intended for it (e.g. coming - * from or addressed to different physical ports, VFs or - * applications) at the device level. - * - * It complements the behavior of some pattern items such as - * RTE_FLOW_ITEM_TYPE_PHY_PORT and is meaningless without them. - * - * When transferring flow rules, ingress and egress attributes keep - * their original meaning, as if processing traffic emitted or - * received by the application. - */ - uint32_t transfer:1; - struct mlx5_meter_domains_infos *mfts; - /**< Flow table created for this meter. */ - struct mlx5_flow_policer_stats policer_stats; - /**< Meter policer statistics. */ - uint32_t ref_cnt; - /**< Use count. */ - uint32_t active_state:1; - /**< Meter state. */ - uint32_t shared:1; - /**< Meter shared or not. */ - struct mlx5_indexed_pool *flow_ipool; - /**< Index pool for flow id. */ -}; - -/* RFC2697 parameter structure. */ -struct mlx5_flow_meter_srtcm_rfc2697_prm { - /* green_saturation_value = cbs_mantissa * 2^cbs_exponent */ - uint32_t cbs_exponent:5; - uint32_t cbs_mantissa:8; - /* cir = 8G * cir_mantissa * 1/(2^cir_exponent) Bytes/Sec */ - uint32_t cir_exponent:5; - uint32_t cir_mantissa:8; - /* yellow _saturation_value = ebs_mantissa * 2^ebs_exponent */ - uint32_t ebs_exponent:5; - uint32_t ebs_mantissa:8; -}; - -/* Flow meter profile structure. */ -struct mlx5_flow_meter_profile { - TAILQ_ENTRY(mlx5_flow_meter_profile) next; - /**< Pointer to the next flow meter structure. */ - uint32_t meter_profile_id; /**< Profile id. */ - struct rte_mtr_meter_profile profile; /**< Profile detail. */ - union { - struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm_prm; - /**< srtcm_rfc2697 struct. */ - }; - uint32_t ref_cnt; /**< Use count. */ }; #define MLX5_MAX_TUNNELS 256 @@ -1094,7 +963,7 @@ struct rte_flow { /**< Device flow handles that are part of the flow. */ uint32_t drv_type:2; /**< Driver type. */ uint32_t tunnel:1; - uint32_t meter:16; /**< Holds flow meter id. */ + uint32_t meter:24; /**< Holds flow meter id. */ uint32_t rix_mreg_copy; /**< Index to metadata register copy table resource. */ uint32_t counter; /**< Holds flow counter. */ @@ -1163,7 +1032,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ - struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ + struct mlx5_flow_meter_info *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1209,12 +1078,16 @@ typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) (struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); typedef int (*mlx5_flow_destroy_policer_rules_t) (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm, + const struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); +typedef uint32_t (*mlx5_flow_mtr_alloc_t) + (struct rte_eth_dev *dev); +typedef void (*mlx5_flow_mtr_free_t)(struct rte_eth_dev *dev, + uint32_t mtr_idx); typedef uint32_t (*mlx5_flow_counter_alloc_t) (struct rte_eth_dev *dev); typedef void (*mlx5_flow_counter_free_t)(struct rte_eth_dev *dev, @@ -1269,6 +1142,8 @@ struct mlx5_flow_driver_ops { mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; + mlx5_flow_mtr_alloc_t create_meter; + mlx5_flow_mtr_free_t free_meter; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; mlx5_flow_counter_query_t counter_query; @@ -1328,6 +1203,32 @@ tunnel_use_standard_attr_group_translate return verdict; } +/** + * Get DV flow aso meter by index. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] idx + * mlx5 flow aso meter index in the container. + * @param[out] ppool + * mlx5 flow aso meter pool in the container, + * + * @return + * Pointer to the aso meter, NULL otherwise. + */ +static inline struct mlx5_aso_mtr * +mlx5_aso_meter_by_idx(struct mlx5_priv *priv, uint32_t idx) +{ + struct mlx5_aso_mtr_pool *pool; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + + /* Decrease to original index. */ + idx--; + MLX5_ASSERT(idx / MLX5_ASO_MTRS_PER_POOL < mtrmng->n); + pool = mtrmng->pools[idx / MLX5_ASO_MTRS_PER_POOL]; + return &pool->mtrs[idx % MLX5_ASO_MTRS_PER_POOL]; +} + int mlx5_flow_group_to_table(struct rte_eth_dev *dev, const struct mlx5_flow_tunnel *tunnel, uint32_t group, uint32_t *table, @@ -1471,10 +1372,10 @@ struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); int mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error); @@ -1570,12 +1471,11 @@ struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev, int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev, const struct rte_flow_item *item, struct rte_flow_error *error); - void flow_release_workspace(void *data); int mlx5_flow_os_init_workspace_once(void); void *mlx5_flow_os_get_specific_workspace(void); int mlx5_flow_os_set_specific_workspace(struct mlx5_flow_workspace *data); void mlx5_flow_os_release_workspace(void); - - +uint32_t mlx5_flow_mtr_alloc(struct rte_eth_dev *dev); +void mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx); #endif /* RTE_PMD_MLX5_FLOW_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 30fec09987..b2178dea18 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -4862,7 +4862,7 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_meter *am = action->conf; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; if (!am) return rte_flow_error_set(error, EINVAL, @@ -4882,7 +4882,7 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "meter action not supported"); - fm = mlx5_flow_meter_find(priv, am->mtr_id); + fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL); if (!fm) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -5924,6 +5924,160 @@ flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter) } } +/** + * Resize a meter id container. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * 0 on success, otherwise negative errno value and rte_errno is set. + */ +static int +flow_dv_mtr_container_resize(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + void *old_pools = mtrmng->pools; + uint32_t resize = mtrmng->n + MLX5_MTRS_CONTAINER_RESIZE; + uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize; + void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); + + if (!pools) { + rte_errno = ENOMEM; + return -ENOMEM; + } + if (old_pools) + memcpy(pools, old_pools, mtrmng->n * + sizeof(struct mlx5_aso_mtr_pool *)); + mtrmng->n = resize; + mtrmng->pools = pools; + if (old_pools) + mlx5_free(old_pools); + return 0; +} + +/** + * Prepare a new meter and/or a new meter pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] mtr_free + * Where to put the pointer of a new meter.g. + * + * @return + * The meter pool pointer and @mtr_free is set on success, + * NULL otherwise and rte_errno is set. + */ +static struct mlx5_aso_mtr_pool * +flow_dv_mtr_pool_create(struct rte_eth_dev *dev, + struct mlx5_aso_mtr **mtr_free) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr_pool *pool = NULL; + struct mlx5_devx_obj *dcs = NULL; + uint32_t i; + uint32_t log_obj_size; + + log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); + dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx, + priv->sh->pdn, log_obj_size); + if (!dcs) { + rte_errno = ENODATA; + return NULL; + } + pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); + if (!pool) { + rte_errno = ENOMEM; + claim_zero(mlx5_devx_cmd_destroy(dcs)); + return NULL; + } + pool->devx_obj = dcs; + pool->index = mtrmng->n_valid; + if (pool->index == mtrmng->n && flow_dv_mtr_container_resize(dev)) { + mlx5_free(pool); + claim_zero(mlx5_devx_cmd_destroy(dcs)); + return NULL; + } + mtrmng->pools[pool->index] = pool; + mtrmng->n_valid++; + for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) { + pool->mtrs[i].offset = i; + pool->mtrs[i].fm.meter_id = UINT32_MAX; + LIST_INSERT_HEAD(&mtrmng->meters, + &pool->mtrs[i], next); + } + pool->mtrs[0].offset = 0; + pool->mtrs[0].fm.meter_id = UINT32_MAX; + *mtr_free = &pool->mtrs[0]; + return pool; +} + +/** + * Release a flow meter into pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_idx + * Index to aso flow meter. + */ +static void +flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); + + MLX5_ASSERT(aso_mtr); + rte_spinlock_lock(&mtrmng->mtrsl); + aso_mtr->state = ASO_METER_FREE; + aso_mtr->fm.meter_id = UINT32_MAX; + LIST_INSERT_HEAD(&mtrmng->meters, aso_mtr, next); + rte_spinlock_unlock(&mtrmng->mtrsl); +} + +/** + * Allocate a aso flow meter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * Index to aso flow meter on success, 0 otherwise and rte_errno is set. + */ +static uint32_t +flow_dv_mtr_alloc(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr *mtr_free = NULL; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr_pool *pool; + uint32_t mtr_idx = 0; + + if (!priv->config.devx) { + rte_errno = ENOTSUP; + return 0; + } + /* Allocate the flow meter memory. */ + /* Get free meters from management. */ + rte_spinlock_lock(&mtrmng->mtrsl); + mtr_free = LIST_FIRST(&mtrmng->meters); + if (mtr_free) + LIST_REMOVE(mtr_free, next); + if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return 0; + } + mtr_free->state = ASO_METER_WAIT; + rte_spinlock_unlock(&mtrmng->mtrsl); + pool = container_of(mtr_free, + struct mlx5_aso_mtr_pool, + mtrs[mtr_free->offset]); + mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset); + return mtr_idx; +} + /** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. @@ -12516,7 +12670,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm = NULL; + struct mlx5_flow_meter_info *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12527,8 +12681,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], - flow->meter); + fm = flow_dv_meter_find_by_idx(priv, flow->meter); if (fm) mlx5_flow_meter_detach(fm); flow->meter = 0; @@ -13547,7 +13700,7 @@ flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, */ static int flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm, + const struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; @@ -13570,6 +13723,8 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. + * @param[in] mtr_idx + * meter index. * @param[in] mtb * Pointer to DV meter table set. * @param[out] drop_rule @@ -13582,7 +13737,8 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, */ static int flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, + uint32_t mtr_idx, struct mlx5_meter_domain_info *dtb, void **drop_rule, void **green_rule) @@ -13629,7 +13785,7 @@ flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, /* Create Drop flow, matching meter_id only. */ i = 0; flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, - (fm->idx << mtr_id_offset), UINT32_MAX); + (mtr_idx << mtr_id_offset), UINT32_MAX); if (mtb->drop_count) actions[i++] = mtb->drop_count; actions[i++] = priv->sh->esw_drop_action; @@ -13643,7 +13799,7 @@ flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, i = 0; if (priv->mtr_reg_share) { flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, - ((fm->idx << mtr_id_offset) | + ((mtr_idx << mtr_id_offset) | rte_col_2_mlx5_col(RTE_COLOR_GREEN)), UINT32_MAX); } else { @@ -13651,7 +13807,7 @@ flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, rte_col_2_mlx5_col(RTE_COLOR_GREEN), UINT32_MAX); flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, - fm->idx, UINT32_MAX); + mtr_idx, UINT32_MAX); } if (mtb->green_count) actions[i++] = mtb->green_count; @@ -13684,9 +13840,10 @@ flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, */ static int flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; bool initialized = false; struct mlx5_flow_counter *cnt; @@ -13696,6 +13853,7 @@ flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, void *ingress_green_rule = NULL; void *transfer_drop_rule = NULL; void *transfer_green_rule = NULL; + uint32_t mtr_idx; int ret; /* Get the statistics counters for green/drop. */ @@ -13722,9 +13880,23 @@ flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, */ if (mtb->egress.drop_rule) initialized = true; + if (priv->sh->meter_aso_en) { + struct mlx5_aso_mtr *aso_mtr = NULL; + struct mlx5_aso_mtr_pool *pool; + + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); + pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool, + mtrs[aso_mtr->offset]); + mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, aso_mtr->offset); + } else { + struct mlx5_legacy_flow_meter *legacy_fm; + + legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm); + mtr_idx = legacy_fm->idx; + } if (attr->egress) { ret = flow_dv_create_policer_forward_rule(dev, - fm, &mtb->egress, + fm, mtr_idx, &mtb->egress, &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); @@ -13733,7 +13905,7 @@ flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, } if (attr->ingress) { ret = flow_dv_create_policer_forward_rule(dev, - fm, &mtb->ingress, + fm, mtr_idx, &mtb->ingress, &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); @@ -13742,7 +13914,7 @@ flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, } if (attr->transfer) { ret = flow_dv_create_policer_forward_rule(dev, - fm, &mtb->transfer, + fm, mtr_idx, &mtb->transfer, &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); @@ -14081,6 +14253,8 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, + .create_meter = flow_dv_mtr_alloc, + .free_meter = flow_dv_aso_mtr_release_to_pool, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, .counter_query = flow_dv_counter_query, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index 73b6e771d9..bbfc8d885a 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -28,40 +28,43 @@ */ static void * mlx5_flow_meter_action_create(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm) + struct mlx5_flow_meter_info *fm) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER struct mlx5dv_dr_flow_meter_attr mtr_init; - void *attr = fm->mfts->fmp; + uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fm->profile->srtcm_prm; + uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); + uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); + uint32_t val; - fm->mfts->fmp_size = MLX5_ST_SZ_BYTES(flow_meter_parameters); - memset(attr, 0, fm->mfts->fmp_size); - MLX5_SET(flow_meter_parameters, attr, valid, 1); - MLX5_SET(flow_meter_parameters, attr, bucket_overflow, 1); - MLX5_SET(flow_meter_parameters, attr, - start_color, MLX5_FLOW_COLOR_GREEN); - MLX5_SET(flow_meter_parameters, attr, both_buckets_on_green, 0); - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); + memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); + MLX5_SET(flow_meter_parameters, fmp, valid, 1); + MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); + MLX5_SET(flow_meter_parameters, fmp, + start_color, MLX5_FLOW_COLOR_GREEN); + MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); + val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); + val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); + val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); + val = (cbs_cir & ASO_DSEG_MAN_MASK); + MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); + val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); + val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); mtr_init.next_table = fm->transfer ? fm->mfts->transfer.tbl->obj : - fm->egress ? fm->mfts->egress.tbl->obj : - fm->mfts->ingress.tbl->obj; + fm->egress ? fm->mfts->egress.tbl->obj : + fm->mfts->ingress.tbl->obj; mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; - mtr_init.flow_meter_parameter = fm->mfts->fmp; - mtr_init.flow_meter_parameter_sz = fm->mfts->fmp_size; + mtr_init.flow_meter_parameter = fmp; + mtr_init.flow_meter_parameter_sz = + MLX5_ST_SZ_BYTES(flow_meter_parameters); mtr_init.active = fm->active_state; return mlx5_glue->dv_create_flow_action_meter(&mtr_init); #else @@ -89,7 +92,7 @@ mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) struct mlx5_flow_meter_profile *fmp; TAILQ_FOREACH(fmp, fmps, next) - if (meter_profile_id == fmp->meter_profile_id) + if (meter_profile_id == fmp->id) return fmp; return NULL; } @@ -239,44 +242,51 @@ mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, { struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; uint8_t man, exp; + uint32_t cbs_exp, cbs_man, cir_exp, cir_man; + uint32_t ebs_exp, ebs_man; if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697) return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, "Metering algorithm not supported."); + /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ + mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir, + &man, &exp); + /* Check if cir mantissa is too large. */ + if (exp > ASO_DSEG_CIR_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, + "meter profile parameter cir is" + " not supported."); + cir_man = man; + cir_exp = exp; /* cbs = cbs_mantissa * 2^cbs_exponent */ mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.cbs, &man, &exp); - srtcm->cbs_mantissa = man; - srtcm->cbs_exponent = exp; /* Check if cbs mantissa is too large. */ - if (srtcm->cbs_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, + if (exp > ASO_DSEG_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter cbs is" - " invalid."); - /* ebs = ebs_mantissa * 2^ebs_exponent */ + "meter profile parameter cbs is" + " not supported."); + cbs_man = man; + cbs_exp = exp; + srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | + cbs_man << ASO_DSEG_CBS_MAN_OFFSET | + cir_exp << ASO_DSEG_CIR_EXP_OFFSET | + cir_man); mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.ebs, &man, &exp); - srtcm->ebs_mantissa = man; - srtcm->ebs_exponent = exp; /* Check if ebs mantissa is too large. */ - if (srtcm->ebs_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, - RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter ebs is" - " invalid."); - /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ - mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir, - &man, &exp); - srtcm->cir_mantissa = man; - srtcm->cir_exponent = exp; - /* Check if cir mantissa is too large. */ - if (srtcm->cir_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, + if (exp > ASO_DSEG_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter cir is" - " invalid."); + "meter profile parameter ebs is" + " not supported."); + ebs_man = man; + ebs_exp = exp; + srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | + ebs_man << ASO_DSEG_EBS_MAN_OFFSET); return 0; } @@ -306,7 +316,11 @@ mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); memset(cap, 0, sizeof(*cap)); - cap->n_max = 1 << qattr->log_max_flow_meter; + if (priv->sh->meter_aso_en) + /* 2 meters per one ASO cache line. */ + cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); + else + cap->n_max = 1 << qattr->log_max_flow_meter; cap->n_shared_max = cap->n_max; cap->identical = 1; cap->shared_identical = 1; @@ -365,7 +379,7 @@ mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, NULL, "Meter profile memory " "alloc failed."); /* Fill profile info. */ - fmp->meter_profile_id = meter_profile_id; + fmp->id = meter_profile_id; fmp->profile = *profile; /* Fill the flow meter parameters for the PRM. */ ret = mlx5_flow_meter_param_fill(fmp, error); @@ -499,7 +513,7 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, NULL, "Green color only supports recolor green action."); /* Validate meter id. */ - if (mlx5_flow_meter_find(priv, meter_id)) + if (mlx5_flow_meter_find(priv, meter_id, NULL)) return -rte_mtr_error_set(error, EEXIST, RTE_MTR_ERROR_TYPE_MTR_ID, NULL, "Meter object already exists."); @@ -524,7 +538,7 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, */ static int mlx5_flow_meter_action_modify(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, uint64_t modify_bits, uint32_t active_state) { @@ -533,33 +547,37 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, uint32_t *attr; struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; int ret; + uint32_t cbs_cir, ebs_eir, val; /* Fill command parameters. */ mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; mod_attr.flow_meter_parameter = in; - mod_attr.flow_meter_parameter_sz = fm->mfts->fmp_size; + mod_attr.flow_meter_parameter_sz = + MLX5_ST_SZ_BYTES(flow_meter_parameters); if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) mod_attr.active = !!active_state; else mod_attr.active = 0; attr = in; + cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); + ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); + val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, cbs_exponent, val); + val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, cbs_mantissa, val); } if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); + val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, cir_exponent, val); + val = cbs_cir & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, cir_mantissa, val); } if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); + val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, ebs_exponent, val); + val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, ebs_mantissa, val); } /* Apply modifications to meter only if it was created. */ if (fm->mfts->meter_action) { @@ -572,26 +590,6 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, /* Update succeedded modify meter parameters. */ if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) fm->active_state = !!active_state; - attr = fm->mfts->fmp; - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); - } - return 0; #else (void)priv; @@ -604,7 +602,7 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, } static void -mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter *fm, +mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter_info *fm, uint64_t stats_mask) { fm->green_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_GREEN) ? 1 : 0; @@ -639,9 +637,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, @@ -653,8 +652,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .need_lock = 1, .type = "mlx5_flow_mtr_flow_id_pool", }; + struct mlx5_aso_mtr *aso_mtr; + uint32_t mtr_idx; int ret; - uint32_t idx = 0; uint8_t mtr_id_bits; uint8_t mtr_reg_bits = priv->mtr_reg_share ? MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; @@ -674,19 +674,31 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, NULL, "Meter profile id not valid."); /* Allocate the flow meter memory. */ - fm = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MTR], &idx); - if (fm == NULL) - return -rte_mtr_error_set(error, ENOMEM, - RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, - "Memory alloc failed for meter."); - mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if (priv->sh->meter_aso_en) { + mtr_idx = mlx5_flow_mtr_alloc(dev); + if (!mtr_idx) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Memory alloc failed for meter."); + aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); + fm = &aso_mtr->fm; + } else { + legacy_fm = mlx5_ipool_zmalloc + (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); + if (legacy_fm == NULL) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Memory alloc failed for meter."); + legacy_fm->idx = mtr_idx; + fm = &legacy_fm->fm; + } + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { DRV_LOG(ERR, "Meter number exceeds max limit."); goto error; } if (mtr_id_bits > priv->max_mtr_bits) priv->max_mtr_bits = mtr_id_bits; - fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; fm->profile = fmp; @@ -712,10 +724,11 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (ret) goto error; /* Add to the flow meter list. */ - TAILQ_INSERT_TAIL(fms, fm, next); + if (!priv->sh->meter_aso_en) + TAILQ_INSERT_TAIL(fms, legacy_fm, next); fm->active_state = 1; /* Config meter starts as active. */ fm->shared = !!shared; - fm->profile->ref_cnt++; + __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); if (!fm->flow_ipool) goto error; @@ -729,12 +742,57 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, mlx5_counter_free(dev, fm->policer_stats.pass_cnt); if (fm->policer_stats.drop_cnt) mlx5_counter_free(dev, fm->policer_stats.drop_cnt); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], idx); + if (priv->sh->meter_aso_en) + mlx5_flow_mtr_free(dev, mtr_idx); + else + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); return -rte_mtr_error_set(error, -ret, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Failed to create devx meter."); } +static int +mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm, + const struct rte_flow_attr *attr, + uint32_t mtr_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; + struct mlx5_flow_meter_profile *fmp; + struct mlx5_legacy_flow_meter *legacy_fm = NULL; + + /* Meter object must not have any owner. */ + MLX5_ASSERT(!fm->ref_cnt); + /* Get meter profile. */ + fmp = fm->profile; + if (fmp == NULL) + return -1; + /* Update dependencies. */ + __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); + /* Remove from list. */ + if (!priv->sh->meter_aso_en) { + legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm); + TAILQ_REMOVE(fms, legacy_fm, next); + } + /* Free policer counters. */ + if (fm->policer_stats.pass_cnt) + mlx5_counter_free(dev, fm->policer_stats.pass_cnt); + if (fm->policer_stats.drop_cnt) + mlx5_counter_free(dev, fm->policer_stats.drop_cnt); + /* Free meter flow table. */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); + mlx5_flow_destroy_policer_rules(dev, fm, attr); + mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); + if (priv->sh->meter_aso_en) + mlx5_flow_mtr_free(dev, mtr_idx); + else + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], + legacy_fm->idx); + return 0; +} + /** * Destroy meter rules. * @@ -753,21 +811,21 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; - struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + uint32_t mtr_idx = 0; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -777,24 +835,17 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, EBUSY, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter object is being used."); - /* Get the meter profile. */ - fmp = fm->profile; - MLX5_ASSERT(fmp); - /* Update dependencies. */ - fmp->ref_cnt--; - /* Remove from the flow meter list. */ - TAILQ_REMOVE(fms, fm, next); - /* Free policer counters. */ - if (fm->policer_stats.pass_cnt) - mlx5_counter_free(dev, fm->policer_stats.pass_cnt); - if (fm->policer_stats.drop_cnt) - mlx5_counter_free(dev, fm->policer_stats.drop_cnt); - /* Free meter flow table */ - if (fm->flow_ipool) - mlx5_ipool_destroy(fm->flow_ipool); - mlx5_flow_destroy_policer_rules(dev, fm, &attr); - mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); + if (priv->sh->meter_aso_en) { + if (mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id)) + return -rte_mtr_error_set(error, EBUSY, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Fail to delete ASO Meter in index table."); + } + /* Destroy the meter profile. */ + if (mlx5_flow_meter_params_flush(dev, fm, &attr, mtr_idx)) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); return 0; } @@ -815,17 +866,13 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, */ static int mlx5_flow_meter_modify_state(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, uint32_t new_state, struct rte_mtr_error *error) { static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { - .cbs_exponent = 20, - .cbs_mantissa = 191, - .cir_exponent = 0, - .cir_mantissa = 200, - .ebs_exponent = 0, - .ebs_mantissa = 0, + .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), + .ebs_eir = 0, }; uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; @@ -867,7 +914,7 @@ mlx5_flow_meter_enable(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; int ret; if (!priv->mtr_en) @@ -875,7 +922,7 @@ mlx5_flow_meter_enable(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -908,7 +955,7 @@ mlx5_flow_meter_disable(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; int ret; if (!priv->mtr_en) @@ -916,7 +963,7 @@ mlx5_flow_meter_disable(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -954,7 +1001,7 @@ mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_meter_profile *fmp; struct mlx5_flow_meter_profile *old_fmp; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; int ret; @@ -970,7 +1017,7 @@ mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, NULL, "Meter profile not found."); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -1020,7 +1067,7 @@ mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, @@ -1034,7 +1081,7 @@ mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -1134,7 +1181,7 @@ mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; struct mlx5_flow_policer_stats *ps; uint64_t pkts; uint64_t bytes; @@ -1145,7 +1192,7 @@ mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -1239,18 +1286,68 @@ mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) * @return * Pointer to the profile found on success, NULL otherwise. */ -struct mlx5_flow_meter * -mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) +struct mlx5_flow_meter_info * +mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, + uint32_t *mtr_idx) { - struct mlx5_flow_meters *fms = &priv->flow_meters; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; + struct mlx5_aso_mtr *aso_mtr; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + union mlx5_l3t_data data; - TAILQ_FOREACH(fm, fms, next) - if (meter_id == fm->meter_id) - return fm; + if (priv->sh->meter_aso_en) { + rte_spinlock_lock(&mtrmng->mtrsl); + if (!mtrmng->n_valid) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return NULL; + } + if (mlx5_l3t_get_entry(mtrmng->mtr_idx_tbl, meter_id, &data) || + !data.dword) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return NULL; + } + if (mtr_idx) + *mtr_idx = data.dword; + aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); + mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id); + if (meter_id == aso_mtr->fm.meter_id) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return &aso_mtr->fm; + } + rte_spinlock_unlock(&mtrmng->mtrsl); + } else { + TAILQ_FOREACH(legacy_fm, fms, next) + if (meter_id == legacy_fm->fm.meter_id) + return &legacy_fm->fm; + } return NULL; } +/** + * Find meter by index. + * + * @param priv + * Pointer to mlx5_priv. + * @param idx + * Meter index. + * + * @return + * Pointer to the profile found on success, NULL otherwise. + */ +struct mlx5_flow_meter_info * +flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) +{ + struct mlx5_aso_mtr *aso_mtr; + + if (priv->sh->meter_aso_en) { + aso_mtr = mlx5_aso_meter_by_idx(priv, idx); + return &aso_mtr->fm; + } else { + return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); + } +} + /** * Attach meter to flow. * Unidirectional Meter creation can only be done @@ -1270,7 +1367,7 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) */ int mlx5_flow_meter_attach(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { @@ -1319,7 +1416,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, * Pointer to flow meter. */ void -mlx5_flow_meter_detach(struct mlx5_flow_meter *fm) +mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER rte_spinlock_lock(&fm->sl); @@ -1352,39 +1449,46 @@ int mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_flow_meter_info *fm; + struct mlx5_aso_mtr_pool *mtr_pool; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; void *tmp; + uint32_t i, offset, mtr_idx; - TAILQ_FOREACH_SAFE(fm, fms, next, tmp) { - /* Meter object must not have any owner. */ - MLX5_ASSERT(!fm->ref_cnt); - /* Get meter profile. */ - fmp = fm->profile; - if (fmp == NULL) - return -rte_mtr_error_set(error, EINVAL, - RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, - NULL, "MTR object meter profile invalid."); - /* Update dependencies. */ - fmp->ref_cnt--; - /* Remove from list. */ - TAILQ_REMOVE(fms, fm, next); - /* Free policer counters. */ - if (fm->policer_stats.pass_cnt) - mlx5_counter_free(dev, fm->policer_stats.pass_cnt); - if (fm->policer_stats.drop_cnt) - mlx5_counter_free(dev, fm->policer_stats.drop_cnt); - /* Free meter flow table. */ - mlx5_flow_destroy_policer_rules(dev, fm, &attr); - mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); + if (priv->sh->meter_aso_en) { + i = mtrmng->n_valid; + while (i--) { + mtr_pool = mtrmng->pools[i]; + for (offset = 0; offset < MLX5_ASO_MTRS_PER_POOL; + offset++) { + fm = &mtr_pool->mtrs[offset].fm; + mtr_idx = MLX5_MAKE_MTR_IDX(i, offset); + if (fm->meter_id != UINT32_MAX && + mlx5_flow_meter_params_flush(dev, + fm, &attr, mtr_idx)) + return -rte_mtr_error_set + (error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); + } + } + } else { + TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { + fm = &legacy_fm->fm; + if (mlx5_flow_meter_params_flush(dev, fm, &attr, 0)) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); + } } TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) { /* Check unused. */ From patchwork Wed Mar 31 07:36:27 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90179 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id CA91DA034F; Wed, 31 Mar 2021 09:37:43 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id F2450140E35; Wed, 31 Mar 2021 09:36:56 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 9F790140E01 for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:42 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwH001778; Wed, 31 Mar 2021 10:36:42 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:27 +0300 Message-Id: <20210331073632.1443011-10-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 09/13] net/mlx5: init/uninit flow meter queue for WQE X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Init/uninit flow meter SQ for WQE Signed-off-by: Li Zhang --- drivers/net/mlx5/linux/mlx5_os.c | 17 ++++ drivers/net/mlx5/meson.build | 2 +- drivers/net/mlx5/mlx5.c | 78 ++++++++++++++- drivers/net/mlx5/mlx5.h | 22 +++-- drivers/net/mlx5/mlx5_flow.h | 4 +- .../mlx5/{mlx5_flow_age.c => mlx5_flow_aso.c} | 96 ++++++++++++++++--- drivers/net/mlx5/mlx5_flow_dv.c | 7 +- drivers/net/mlx5/mlx5_flow_meter.c | 7 +- 8 files changed, 201 insertions(+), 32 deletions(-) rename drivers/net/mlx5/{mlx5_flow_age.c => mlx5_flow_aso.c} (85%) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index b7eca9d153..1794a51073 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -813,6 +813,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, char name[RTE_ETH_NAME_MAX_LEN]; int own_domain_id = 0; uint16_t port_id; + uint32_t log_obj_size; #ifdef HAVE_MLX5DV_DR_DEVX_PORT struct mlx5dv_devx_port devx_port = { .comp_mask = 0 }; #endif @@ -1283,6 +1284,22 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, priv->mtr_color_reg); } } + if (config->hca_attr.qos.sup && + config->hca_attr.qos.flow_meter_aso_sup) { + log_obj_size = + rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); + if (log_obj_size >= + config->hca_attr.qos.log_meter_aso_granularity && + log_obj_size <= + config->hca_attr.qos.log_meter_aso_max_alloc) { + sh->meter_aso_en = 1; + err = mlx5_aso_flow_mtrs_mng_init(priv); + if (err) { + err = -err; + goto error; + } + } + } #endif #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO if (config->hca_attr.flow_hit_aso && diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build index f2fafbdd05..89a16f8f3a 100644 --- a/drivers/net/mlx5/meson.build +++ b/drivers/net/mlx5/meson.build @@ -15,7 +15,7 @@ sources = files( 'mlx5_flow.c', 'mlx5_flow_meter.c', 'mlx5_flow_dv.c', - 'mlx5_flow_age.c', + 'mlx5_flow_aso.c', 'mlx5_mac.c', 'mlx5_mr.c', 'mlx5_rss.c', diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 8e0582826d..d6e27947f5 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -401,7 +401,7 @@ mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) rte_errno = ENOMEM; return -ENOMEM; } - err = mlx5_aso_queue_init(sh); + err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT); if (err) { mlx5_free(sh->aso_age_mng); return -1; @@ -423,8 +423,8 @@ mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) { int i, j; - mlx5_aso_queue_stop(sh); - mlx5_aso_queue_uninit(sh); + mlx5_aso_flow_hit_queue_poll_stop(sh); + mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); if (sh->aso_age_mng->pools) { struct mlx5_aso_age_pool *pool; @@ -562,6 +562,66 @@ mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) memset(&sh->cmng, 0, sizeof(sh->cmng)); } +/** + * Initialize the aso flow meters management structure. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object to free + */ +int +mlx5_aso_flow_mtrs_mng_init(struct mlx5_priv *priv) +{ + if (!priv->mtr_idx_tbl) { + priv->mtr_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); + if (!priv->mtr_idx_tbl) { + DRV_LOG(ERR, "fail to create meter lookup table."); + rte_errno = ENOMEM; + return -ENOMEM; + } + } + if (!priv->sh->mtrmng) { + priv->sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*priv->sh->mtrmng), + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); + if (!priv->sh->mtrmng) { + DRV_LOG(ERR, "mlx5_aso_mtr_pools_mng allocation was failed."); + rte_errno = ENOMEM; + return -ENOMEM; + } + rte_spinlock_init(&priv->sh->mtrmng->mtrsl); + LIST_INIT(&priv->sh->mtrmng->meters); + } + return 0; +} + +/** + * Close and release all the resources of + * the ASO flow meter management structure. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object to free. + */ +static void +mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) +{ + struct mlx5_aso_mtr_pool *mtr_pool; + struct mlx5_aso_mtr_pools_mng *mtrmng = sh->mtrmng; + uint32_t idx; + + mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); + idx = mtrmng->n_valid; + while (idx--) { + mtr_pool = mtrmng->pools[idx]; + claim_zero(mlx5_devx_cmd_destroy + (mtr_pool->devx_obj)); + mtrmng->n_valid--; + mlx5_free(mtr_pool); + } + mlx5_free(sh->mtrmng->pools); + mlx5_free(sh->mtrmng); + sh->mtrmng = NULL; +} + /* Send FLOW_AGED event if needed. */ void mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) @@ -1111,6 +1171,8 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) mlx5_flow_aso_age_mng_close(sh); sh->aso_age_mng = NULL; } + if (sh->mtrmng) + mlx5_aso_flow_mtrs_mng_close(sh); mlx5_flow_ipool_destroy(sh); mlx5_os_dev_shared_handler_uninstall(sh); if (sh->cnt_id_tbl) { @@ -1315,6 +1377,8 @@ mlx5_dev_close(struct rte_eth_dev *dev) struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; int ret; + uint32_t mtr_idx; + void *entry; if (rte_eal_process_type() == RTE_PROC_SECONDARY) { /* Check if process_private released. */ @@ -1391,6 +1455,14 @@ mlx5_dev_close(struct rte_eth_dev *dev) close(priv->nl_socket_rdma); if (priv->vmwa_context) mlx5_vlan_vmwa_exit(priv->vmwa_context); + if (priv->mtr_idx_tbl) { + MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { + mtr_idx = *(uint32_t *)entry; + if (mtr_idx) + mlx5_flow_mtr_free(dev, mtr_idx); + } + mlx5_l3t_destroy(priv->mtr_idx_tbl); + } ret = mlx5_hrxq_verify(dev); if (ret) DRV_LOG(WARNING, "port %u some hash Rx queue still remain", diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index ced14dd0bf..6c8eb6c68c 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -492,8 +492,13 @@ struct mlx5_aso_devx_mr { }; struct mlx5_aso_sq_elem { - struct mlx5_aso_age_pool *pool; - uint16_t burst_size; + union { + struct { + struct mlx5_aso_age_pool *pool; + uint16_t burst_size; + }; + struct mlx5_aso_mtr *mtr; + }; }; struct mlx5_aso_sq { @@ -765,7 +770,6 @@ struct mlx5_aso_mtr_pools_mng { volatile uint16_t n_valid; /* Number of valid pools. */ uint16_t n; /* Number of pools. */ rte_spinlock_t mtrsl; /* The ASO flow meter free list lock. */ - struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */ struct aso_meter_list meters; /* Free ASO flow meter list. */ struct mlx5_aso_sq sq; /*SQ using by ASO flow meter. */ struct mlx5_aso_mtr_pool **pools; /* ASO flow meter pool array. */ @@ -1196,6 +1200,7 @@ struct mlx5_priv { uint8_t mtr_color_reg; /* Meter color match REG_C. */ struct mlx5_mtr_profiles flow_meter_profiles; /* MTR profile list. */ struct mlx5_legacy_flow_meters flow_meters; /* MTR list. */ + struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */ uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */ uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */ struct mlx5_mp_id mp_id; /* ID of a multi-process process */ @@ -1259,6 +1264,7 @@ int mlx5_hairpin_cap_get(struct rte_eth_dev *dev, bool mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev); int mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev); int mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_flow_mtrs_mng_init(struct mlx5_priv *priv); /* mlx5_ethdev.c */ @@ -1529,9 +1535,11 @@ eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev); /* mlx5_flow_aso.c */ -int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh); -int mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh); -int mlx5_aso_queue_stop(struct mlx5_dev_ctx_shared *sh); -void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod); +int mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh); +void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod); #endif /* RTE_PMD_MLX5_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 8debe1c845..9e146685c2 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -827,8 +827,8 @@ struct mlx5_flow { #define MLX5_FLOW_METER_DISABLE 0 #define MLX5_FLOW_METER_ENABLE 1 -#define MLX5_ASO_CQE_RESPONSE_DELAY 10 -#define MLX5_MTR_POLL_CQE_TIMES 100000u +#define MLX5_ASO_WQE_CQE_RESPONSE_DELAY 10u +#define MLX5_MTR_POLL_WQE_CQE_TIMES 100000u #define MLX5_MAN_WIDTH 8 /* Legacy Meter parameter structure. */ diff --git a/drivers/net/mlx5/mlx5_flow_age.c b/drivers/net/mlx5/mlx5_flow_aso.c similarity index 85% rename from drivers/net/mlx5/mlx5_flow_age.c rename to drivers/net/mlx5/mlx5_flow_aso.c index 00cb20dd62..067471ba0f 100644 --- a/drivers/net/mlx5/mlx5_flow_age.c +++ b/drivers/net/mlx5/mlx5_flow_aso.c @@ -144,7 +144,6 @@ mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq) { mlx5_devx_sq_destroy(&sq->sq_obj); mlx5_aso_cq_destroy(&sq->cq); - mlx5_aso_devx_dereg_mr(&sq->mr); memset(sq, 0, sizeof(*sq)); } @@ -155,7 +154,7 @@ mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq) * ASO SQ to initialize. */ static void -mlx5_aso_init_sq(struct mlx5_aso_sq *sq) +mlx5_aso_age_init_sq(struct mlx5_aso_sq *sq) { volatile struct mlx5_aso_wqe *restrict wqe; int i; @@ -181,6 +180,39 @@ mlx5_aso_init_sq(struct mlx5_aso_sq *sq) } } +/** + * Initialize Send Queue used for ASO flow meter access. + * + * @param[in] sq + * ASO SQ to initialize. + */ +static void +mlx5_aso_mtr_init_sq(struct mlx5_aso_sq *sq) +{ + volatile struct mlx5_aso_wqe *restrict wqe; + int i; + int size = 1 << sq->log_desc_n; + uint32_t idx; + + /* All the next fields state should stay constant. */ + for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) { + wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | + (sizeof(*wqe) >> 4)); + wqe->aso_cseg.operand_masks = RTE_BE32(0u | + (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | + (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) | + (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) | + (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); + wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << + MLX5_COMP_MODE_OFFSET); + for (idx = 0; idx < MLX5_ASO_METERS_PER_WQE; + idx++) + wqe->aso_dseg.mtrs[idx].v_bo_sc_bbog_mm = + RTE_BE32((1 << ASO_DSEG_VALID_OFFSET) | + (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET)); + } +} + /** * Create Send Queue used for ASO access. * @@ -216,13 +248,9 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, struct mlx5_devx_modify_sq_attr modify_attr = { .state = MLX5_SQC_STATE_RDY, }; - uint32_t sq_desc_n = 1 << log_desc_n; uint16_t log_wqbb_n; int ret; - if (mlx5_aso_devx_reg_mr(ctx, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * - sq_desc_n, &sq->mr, socket, pdn)) - return -1; if (mlx5_aso_cq_create(ctx, &sq->cq, log_desc_n, socket, mlx5_os_get_devx_uar_page_id(uar))) goto error; @@ -247,7 +275,6 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, sq->tail = 0; sq->sqn = sq->sq_obj.sq->id; sq->uar_addr = mlx5_os_get_devx_uar_reg_addr(uar); - mlx5_aso_init_sq(sq); return 0; error: mlx5_aso_destroy_sq(sq); @@ -264,11 +291,37 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh) +mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod) { - return mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0, + uint32_t sq_desc_n = 1 << MLX5_ASO_QUEUE_LOG_DESC; + + switch (aso_opc_mod) { + case ASO_OPC_MOD_FLOW_HIT: + if (mlx5_aso_devx_reg_mr(sh->ctx, + (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * + sq_desc_n, &sh->aso_age_mng->aso_sq.mr, 0, sh->pdn)) + return -1; + if (mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0, + sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC, + sh->sq_ts_format)) { + mlx5_aso_devx_dereg_mr(&sh->aso_age_mng->aso_sq.mr); + return -1; + } + mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq); + break; + case ASO_OPC_MOD_POLICER: + if (mlx5_aso_sq_create(sh->ctx, &sh->mtrmng->sq, 0, sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC, - sh->sq_ts_format); + sh->sq_ts_format)) + return -1; + mlx5_aso_mtr_init_sq(&sh->mtrmng->sq); + break; + default: + DRV_LOG(ERR, "Unknown ASO operation mode"); + return -1; + } + return 0; } /** @@ -278,9 +331,24 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh) * Pointer to shared device context. */ void -mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh) +mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod) { - mlx5_aso_destroy_sq(&sh->aso_age_mng->aso_sq); + struct mlx5_aso_sq *sq; + + switch (aso_opc_mod) { + case ASO_OPC_MOD_FLOW_HIT: + mlx5_aso_devx_dereg_mr(&sh->aso_age_mng->aso_sq.mr); + sq = &sh->aso_age_mng->aso_sq; + break; + case ASO_OPC_MOD_POLICER: + sq = &sh->mtrmng->sq; + break; + default: + DRV_LOG(ERR, "Unknown ASO operation mode"); + return; + } + mlx5_aso_destroy_sq(sq); } /** @@ -555,7 +623,7 @@ mlx5_flow_aso_alarm(void *arg) * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh) +mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh) { if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) { DRV_LOG(ERR, "Cannot reinitialize ASO age alarm."); @@ -574,7 +642,7 @@ mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh) * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_aso_queue_stop(struct mlx5_dev_ctx_shared *sh) +mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh) { int retries = 1024; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index b2178dea18..03dcc2bfd4 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -5947,6 +5947,11 @@ flow_dv_mtr_container_resize(struct rte_eth_dev *dev) rte_errno = ENOMEM; return -ENOMEM; } + if (!mtrmng->n) + if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) { + mlx5_free(pools); + return -ENOMEM; + } if (old_pools) memcpy(pools, old_pools, mtrmng->n * sizeof(struct mlx5_aso_mtr_pool *)); @@ -10818,7 +10823,7 @@ flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev) mlx5_free(old_pools); } else { /* First ASO flow hit allocation - starting ASO data-path. */ - int ret = mlx5_aso_queue_start(priv->sh); + int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh); if (ret) { mlx5_free(pools); diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index bbfc8d885a..89d1d42e3f 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -811,7 +811,6 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, @@ -836,7 +835,7 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter object is being used."); if (priv->sh->meter_aso_en) { - if (mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id)) + if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id)) return -rte_mtr_error_set(error, EBUSY, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Fail to delete ASO Meter in index table."); @@ -1302,7 +1301,7 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, rte_spinlock_unlock(&mtrmng->mtrsl); return NULL; } - if (mlx5_l3t_get_entry(mtrmng->mtr_idx_tbl, meter_id, &data) || + if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || !data.dword) { rte_spinlock_unlock(&mtrmng->mtrsl); return NULL; @@ -1310,7 +1309,7 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, if (mtr_idx) *mtr_idx = data.dword; aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); - mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id); + mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); if (meter_id == aso_mtr->fm.meter_id) { rte_spinlock_unlock(&mtrmng->mtrsl); return &aso_mtr->fm; From patchwork Wed Mar 31 07:36:28 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90177 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id DE596A034F; Wed, 31 Mar 2021 09:37:29 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 7AFF7140E29; Wed, 31 Mar 2021 09:36:54 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id A7929140E02 for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:42 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwI001778; Wed, 31 Mar 2021 10:36:42 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:28 +0300 Message-Id: <20210331073632.1443011-11-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 10/13] net/mlx5: aso flow meter send WQE and CQE handle X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" ASO flow meter send WQE and CQE handle functions Signed-off-by: Li Zhang --- drivers/net/mlx5/mlx5.h | 4 + drivers/net/mlx5/mlx5_flow_aso.c | 182 +++++++++++++++++++++++++++++ drivers/net/mlx5/mlx5_flow_meter.c | 143 ++++++++++++++--------- 3 files changed, 274 insertions(+), 55 deletions(-) diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 6c8eb6c68c..340ca66c45 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -1541,5 +1541,9 @@ int mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh); int mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh); void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, enum mlx5_access_aso_opc_mod aso_opc_mod); +int mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *mtr); +int mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *mtr); #endif /* RTE_PMD_MLX5_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c index 067471ba0f..4c1b2ed6e5 100644 --- a/drivers/net/mlx5/mlx5_flow_aso.c +++ b/drivers/net/mlx5/mlx5_flow_aso.c @@ -657,3 +657,185 @@ mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh) } return -rte_errno; } + +static uint16_t +mlx5_aso_mtr_sq_enqueue_single(struct mlx5_aso_sq *sq, + struct mlx5_aso_mtr *aso_mtr) +{ + volatile struct mlx5_aso_wqe *wqe = NULL; + struct mlx5_flow_meter_info *fm = NULL; + uint16_t size = 1 << sq->log_desc_n; + uint16_t mask = size - 1; + uint16_t res = size - (uint16_t)(sq->head - sq->tail); + uint32_t dseg_idx = 0; + struct mlx5_aso_mtr_pool *pool = NULL; + + if (unlikely(!res)) { + DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send"); + return 0; + } + wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; + rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); + /* Fill next WQE. */ + fm = &aso_mtr->fm; + sq->elts[sq->head & mask].mtr = aso_mtr; + pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool, + mtrs[aso_mtr->offset]); + wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id + + (aso_mtr->offset >> 1)); + wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO | + (ASO_OPC_MOD_POLICER << + WQE_CSEG_OPC_MOD_OFFSET) | + sq->pi << WQE_CSEG_WQE_INDEX_OFFSET); + /* There are 2 meters in one ASO cache line. */ + dseg_idx = aso_mtr->offset & 0x1; + wqe->aso_cseg.data_mask = + RTE_BE64(MLX5_IFC_FLOW_METER_PARAM_MASK << (32 * !dseg_idx)); + if (fm->is_enable) { + wqe->aso_dseg.mtrs[dseg_idx].cbs_cir = + fm->profile->srtcm_prm.cbs_cir; + wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = + fm->profile->srtcm_prm.ebs_eir; + } else { + wqe->aso_dseg.mtrs[dseg_idx].cbs_cir = + RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL); + wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = 0; + } + sq->head++; + sq->pi += 2;/* Each WQE contains 2 WQEBB's. */ + rte_io_wmb(); + sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi); + rte_wmb(); + *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */ + rte_wmb(); + return 1; +} + +static void +mlx5_aso_mtrs_status_update(struct mlx5_aso_sq *sq, uint16_t aso_mtrs_nums) +{ + uint16_t size = 1 << sq->log_desc_n; + uint16_t mask = size - 1; + uint16_t i; + struct mlx5_aso_mtr *aso_mtr = NULL; + uint8_t exp_state = ASO_METER_WAIT; + + for (i = 0; i < aso_mtrs_nums; ++i) { + aso_mtr = sq->elts[(sq->tail + i) & mask].mtr; + MLX5_ASSERT(aso_mtr); + __atomic_compare_exchange_n(&aso_mtr->state, + &exp_state, ASO_METER_READY, + false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + } +} + +static void +mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq) +{ + struct mlx5_aso_cq *cq = &sq->cq; + volatile struct mlx5_cqe *restrict cqe; + const unsigned int cq_size = 1 << cq->log_desc_n; + const unsigned int mask = cq_size - 1; + uint32_t idx; + uint32_t next_idx = cq->cq_ci & mask; + const uint16_t max = (uint16_t)(sq->head - sq->tail); + uint16_t n = 0; + int ret; + + if (unlikely(!max)) + return; + do { + idx = next_idx; + next_idx = (cq->cq_ci + 1) & mask; + rte_prefetch0(&cq->cq_obj.cqes[next_idx]); + cqe = &cq->cq_obj.cqes[idx]; + ret = check_cqe(cqe, cq_size, cq->cq_ci); + /* + * Be sure owner read is done before any other cookie field or + * opaque field. + */ + rte_io_rmb(); + if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { + if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) + break; + mlx5_aso_cqe_err_handle(sq); + } else { + n++; + } + cq->cq_ci++; + } while (1); + if (likely(n)) { + mlx5_aso_mtrs_status_update(sq, n); + sq->tail += n; + rte_io_wmb(); + cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); + } +} + +/** + * Update meter parameter by send WQE. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] priv + * Pointer to mlx5 private data structure. + * @param[in] fm + * Pointer to flow meter to be modified. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *mtr) +{ + struct mlx5_aso_sq *sq = &sh->mtrmng->sq; + uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; + + do { + mlx5_aso_mtr_completion_handle(sq); + if (mlx5_aso_mtr_sq_enqueue_single(sq, mtr)) + return 0; + /* Waiting for wqe resource. */ + usleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); + } while (--poll_wqe_times); + DRV_LOG(ERR, "Fail to send WQE for ASO meter %d", + mtr->fm.meter_id); + return -1; +} + +/** + * Wait for meter to be ready. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] priv + * Pointer to mlx5 private data structure. + * @param[in] fm + * Pointer to flow meter to be modified. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *mtr) +{ + struct mlx5_aso_sq *sq = &sh->mtrmng->sq; + uint32_t poll_cqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; + + if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) == + ASO_METER_READY) + return 0; + do { + mlx5_aso_mtr_completion_handle(sq); + if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) == + ASO_METER_READY) + return 0; + /* Waiting for CQE ready. */ + usleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); + } while (--poll_cqe_times); + DRV_LOG(ERR, "Fail to poll CQE ready for ASO meter %d", + mtr->fm.meter_id); + return -1; +} diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index 89d1d42e3f..bb36696172 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -540,56 +540,79 @@ static int mlx5_flow_meter_action_modify(struct mlx5_priv *priv, struct mlx5_flow_meter_info *fm, const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, - uint64_t modify_bits, uint32_t active_state) + uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; uint32_t *attr; struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; int ret; + struct mlx5_aso_mtr *aso_mtr = NULL; uint32_t cbs_cir, ebs_eir, val; - /* Fill command parameters. */ - mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; - mod_attr.flow_meter_parameter = in; - mod_attr.flow_meter_parameter_sz = + if (priv->sh->meter_aso_en) { + fm->is_enable = !!is_enable; + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); + ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); + if (ret) + return ret; + ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr); + if (ret) + return ret; + } else { + /* Fill command parameters. */ + mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; + mod_attr.flow_meter_parameter = in; + mod_attr.flow_meter_parameter_sz = MLX5_ST_SZ_BYTES(flow_meter_parameters); - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) - mod_attr.active = !!active_state; - else - mod_attr.active = 0; - attr = in; - cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); - ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { - val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; - MLX5_SET(flow_meter_parameters, attr, cbs_exponent, val); - val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; - MLX5_SET(flow_meter_parameters, attr, cbs_mantissa, val); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { - val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; - MLX5_SET(flow_meter_parameters, attr, cir_exponent, val); - val = cbs_cir & ASO_DSEG_MAN_MASK; - MLX5_SET(flow_meter_parameters, attr, cir_mantissa, val); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { - val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; - MLX5_SET(flow_meter_parameters, attr, ebs_exponent, val); - val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; - MLX5_SET(flow_meter_parameters, attr, ebs_mantissa, val); - } - /* Apply modifications to meter only if it was created. */ - if (fm->mfts->meter_action) { - ret = mlx5_glue->dv_modify_flow_action_meter + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) + mod_attr.active = !!active_state; + else + mod_attr.active = 0; + attr = in; + cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); + ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { + val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & + ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, + cbs_exponent, val); + val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & + ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, + cbs_mantissa, val); + } + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { + val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & + ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, + cir_exponent, val); + val = cbs_cir & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, + cir_mantissa, val); + } + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { + val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & + ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, + ebs_exponent, val); + val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & + ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, + ebs_mantissa, val); + } + /* Apply modifications to meter only if it was created. */ + if (fm->mfts->meter_action) { + ret = mlx5_glue->dv_modify_flow_action_meter (fm->mfts->meter_action, &mod_attr, rte_cpu_to_be_64(modify_bits)); - if (ret) - return ret; + if (ret) + return ret; + } + /* Update succeedded modify meter parameters. */ + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) + fm->active_state = !!active_state; } - /* Update succeedded modify meter parameters. */ - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) - fm->active_state = !!active_state; return 0; #else (void)priv; @@ -653,6 +676,7 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .type = "mlx5_flow_mtr_flow_id_pool", }; struct mlx5_aso_mtr *aso_mtr; + union mlx5_l3t_data data; uint32_t mtr_idx; int ret; uint8_t mtr_id_bits; @@ -704,7 +728,6 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->profile = fmp; memcpy(fm->action, params->action, sizeof(params->action)); mlx5_flow_meter_stats_enable_update(fm, params->stats_mask); - /* Alloc policer counters. */ if (fm->green_bytes || fm->green_pkts) { fm->policer_stats.pass_cnt = mlx5_counter_alloc(dev); @@ -727,12 +750,23 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!priv->sh->meter_aso_en) TAILQ_INSERT_TAIL(fms, legacy_fm, next); fm->active_state = 1; /* Config meter starts as active. */ + fm->is_enable = 1; fm->shared = !!shared; __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); if (!fm->flow_ipool) goto error; rte_spinlock_init(&fm->sl); + /* If ASO meter supported, allocate ASO flow meter. */ + if (priv->sh->meter_aso_en) { + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); + ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); + if (ret) + goto error; + data.dword = mtr_idx; + if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) + goto error; + } return 0; error: mlx5_flow_destroy_policer_rules(dev, fm, &attr); @@ -878,12 +912,12 @@ mlx5_flow_meter_modify_state(struct mlx5_priv *priv, int ret; if (new_state == MLX5_FLOW_METER_DISABLE) - ret = mlx5_flow_meter_action_modify(priv, fm, &srtcm, - modify_bits, 0); + ret = mlx5_flow_meter_action_modify(priv, fm, + &srtcm, modify_bits, 0, 0); else ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, - modify_bits, 0); + modify_bits, 0, 1); if (ret) return -rte_mtr_error_set(error, -ret, RTE_MTR_ERROR_TYPE_MTR_PARAMS, @@ -1031,7 +1065,7 @@ mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, if (fm->active_state == MLX5_FLOW_METER_DISABLE) return 0; ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, - modify_bits, fm->active_state); + modify_bits, fm->active_state, 1); if (ret) { fm->profile = old_fmp; return -rte_mtr_error_set(error, -ret, @@ -1281,6 +1315,8 @@ mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) * Pointer to mlx5_priv. * @param meter_id * Meter id. + * @param mtr_idx + * Pointer to Meter index. * * @return * Pointer to the profile found on success, NULL otherwise. @@ -1297,10 +1333,6 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, if (priv->sh->meter_aso_en) { rte_spinlock_lock(&mtrmng->mtrsl); - if (!mtrmng->n_valid) { - rte_spinlock_unlock(&mtrmng->mtrsl); - return NULL; - } if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || !data.dword) { rte_spinlock_unlock(&mtrmng->mtrsl); @@ -1309,17 +1341,18 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, if (mtr_idx) *mtr_idx = data.dword; aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); + /* Remove reference taken by the mlx5_l3t_get_entry. */ mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); - if (meter_id == aso_mtr->fm.meter_id) { - rte_spinlock_unlock(&mtrmng->mtrsl); - return &aso_mtr->fm; - } + MLX5_ASSERT(meter_id == aso_mtr->fm.meter_id); rte_spinlock_unlock(&mtrmng->mtrsl); - } else { - TAILQ_FOREACH(legacy_fm, fms, next) - if (meter_id == legacy_fm->fm.meter_id) - return &legacy_fm->fm; + return &aso_mtr->fm; } + TAILQ_FOREACH(legacy_fm, fms, next) + if (meter_id == legacy_fm->fm.meter_id) { + if (mtr_idx) + *mtr_idx = legacy_fm->idx; + return &legacy_fm->fm; + } return NULL; } From patchwork Wed Mar 31 07:36:29 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90178 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id B7894A034F; Wed, 31 Mar 2021 09:37:36 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id C2330140E30; Wed, 31 Mar 2021 09:36:55 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id ABA93140E03 for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:42 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwJ001778; Wed, 31 Mar 2021 10:36:42 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com, Shun Hao Date: Wed, 31 Mar 2021 10:36:29 +0300 Message-Id: <20210331073632.1443011-12-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 11/13] net/mlx5: add support of ASO meter action X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" When ASO action is available, use it as the meter action Signed-off-by: Shun Hao Signed-off-by: Li Zhang --- drivers/net/mlx5/linux/mlx5_os.c | 7 ++- drivers/net/mlx5/mlx5.c | 8 +++ drivers/net/mlx5/mlx5.h | 7 ++- drivers/net/mlx5/mlx5_flow.c | 69 +++++++++++++++------ drivers/net/mlx5/mlx5_flow_dv.c | 26 ++++++-- drivers/net/mlx5/mlx5_flow_meter.c | 97 +++++++++++++++++++----------- 6 files changed, 153 insertions(+), 61 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 1794a51073..6713701469 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -813,7 +813,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, char name[RTE_ETH_NAME_MAX_LEN]; int own_domain_id = 0; uint16_t port_id; - uint32_t log_obj_size; #ifdef HAVE_MLX5DV_DR_DEVX_PORT struct mlx5dv_devx_port devx_port = { .comp_mask = 0 }; #endif @@ -1249,7 +1248,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, "required for coalescing is %d bytes", config->hca_attr.lro_min_mss_size); } -#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) +#if defined(HAVE_MLX5DV_DR) && \ + (defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) || \ + defined(HAVE_MLX5_DR_CREATE_ACTION_ASO)) if (config->hca_attr.qos.sup && config->hca_attr.qos.flow_meter_old && config->dv_flow_en) { @@ -1286,7 +1287,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, } if (config->hca_attr.qos.sup && config->hca_attr.qos.flow_meter_aso_sup) { - log_obj_size = + uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); if (log_obj_size >= config->hca_attr.qos.log_meter_aso_granularity && diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index d6e27947f5..f321dd5e18 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -605,13 +605,21 @@ static void mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) { struct mlx5_aso_mtr_pool *mtr_pool; + struct mlx5_aso_mtr *aso_mtr; struct mlx5_aso_mtr_pools_mng *mtrmng = sh->mtrmng; uint32_t idx; + int i; mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); idx = mtrmng->n_valid; while (idx--) { mtr_pool = mtrmng->pools[idx]; + for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { + aso_mtr = &mtr_pool->mtrs[i]; + if (aso_mtr->fm.meter_action) + claim_zero(mlx5_glue->destroy_flow_action + (aso_mtr->fm.meter_action)); + } claim_zero(mlx5_devx_cmd_destroy (mtr_pool->devx_obj)); mtrmng->n_valid--; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 340ca66c45..d0cd402495 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -635,8 +635,6 @@ struct mlx5_meter_domains_infos { /**< Counters for green rule. */ void *drop_count; /**< Counters for green rule. */ - void *meter_action; - /**< Flow meter action. */ }; /* Meter parameter structure. */ @@ -699,6 +697,8 @@ struct mlx5_flow_meter_info { /**< Use count. */ struct mlx5_indexed_pool *flow_ipool; /**< Index pool for flow id. */ + void *meter_action; + /**< Flow meter action. */ }; /* RFC2697 parameter structure. */ @@ -1488,7 +1488,8 @@ int mlx5_flow_meter_attach(struct mlx5_priv *priv, struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error); -void mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm); +void mlx5_flow_meter_detach(struct mlx5_priv *priv, + struct mlx5_flow_meter_info *fm); /* mlx5_os.c */ struct rte_pci_driver; diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 3fe37b92ee..f2dfe85119 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -4379,6 +4379,8 @@ flow_create_split_inner(struct rte_eth_dev *dev, * Parent flow structure pointer. * @param[in] fm * Pointer to flow meter structure. + * @param[in] attr + * Flow rule attributes. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4399,6 +4401,7 @@ static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, struct rte_flow *flow, struct mlx5_flow_meter_info *fm, + const struct rte_flow_attr *attr, const struct rte_flow_item items[], struct rte_flow_item sfx_items[], const struct rte_flow_action actions[], @@ -4416,6 +4419,12 @@ flow_meter_split_prep(struct rte_eth_dev *dev, struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + struct rte_flow_action *hw_mtr_action; + struct rte_flow_action_jump *jump_data; + struct rte_flow_action *action_pre_head = NULL; + bool mtr_first = priv->sh->meter_aso_en && + (attr->egress || + (attr->transfer && priv->representor_id != -1)); uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; uint8_t mtr_reg_bits = priv->mtr_reg_share ? MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; @@ -4423,29 +4432,39 @@ flow_meter_split_prep(struct rte_eth_dev *dev, int shift; uint32_t flow_id_val = 0; + /* For ASO meter, meter must be before tag in TX direction. */ + if (mtr_first) { + action_pre_head = actions_pre++; + /* Leave space for tag action. */ + tag_action = actions_pre++; + } /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { - struct rte_flow_action **action_cur = NULL; + struct rte_flow_action *action_cur = NULL; switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - /* Add the extra tag action first. */ - tag_action = actions_pre++; - action_cur = &actions_pre; + if (mtr_first) { + action_cur = action_pre_head; + } else { + /* Leave space for tag action. */ + tag_action = actions_pre++; + action_cur = actions_pre++; + } break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: raw_encap = actions->conf; if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_RAW_DECAP: raw_decap = actions->conf; if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: @@ -4455,14 +4474,30 @@ flow_meter_split_prep(struct rte_eth_dev *dev, break; } if (!action_cur) - action_cur = &actions_sfx; - memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); - (*action_cur)++; + action_cur = actions_sfx++; + memcpy(action_cur, actions, sizeof(struct rte_flow_action)); } /* Add end action to the actions. */ actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; - actions_pre->type = RTE_FLOW_ACTION_TYPE_END; - actions_pre++; + if (priv->sh->meter_aso_en) { + /** For ASO meter, need to add an extra jump action explicitly, + * to jump from meter to policer table. + */ + hw_mtr_action = actions_pre; + hw_mtr_action->type = RTE_FLOW_ACTION_TYPE_JUMP; + actions_pre++; + actions_pre->type = RTE_FLOW_ACTION_TYPE_END; + actions_pre++; + jump_data = (struct rte_flow_action_jump *)actions_pre; + jump_data->group = attr->transfer ? + (MLX5_FLOW_TABLE_LEVEL_METER - 1) : + MLX5_FLOW_TABLE_LEVEL_METER; + hw_mtr_action->conf = jump_data; + actions_pre = (struct rte_flow_action *)(jump_data + 1); + } else { + actions_pre->type = RTE_FLOW_ACTION_TYPE_END; + actions_pre++; + } /* Generate meter flow_id only if support multiple flows per meter. */ mlx5_ipool_malloc(fm->flow_ipool, &tag_id); if (!tag_id) @@ -5278,10 +5313,10 @@ flow_create_split_meter(struct rte_eth_dev *dev, flow->meter = mtr_idx; } wks->fm = fm; - /* The prefix actions: meter, decap, encap, tag, end. */ - act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + + /* Prefix actions: meter, decap, encap, tag, jump, end. */ + act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* The suffix items: tag, vlan, port id, end. */ + /* Suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5295,8 +5330,8 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, flow, fm, items, - sfx_items, actions, + mtr_tag_id = flow_meter_split_prep(dev, flow, fm, &sfx_attr, + items, sfx_items, actions, sfx_actions, pre_actions, error); if (!mtr_tag_id) { diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 03dcc2bfd4..58bd279240 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -4887,9 +4887,11 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Meter not found"); - if (fm->ref_cnt && (!(fm->transfer == attr->transfer || + /* aso meter can always be shared by different domains */ + if (fm->ref_cnt && !priv->sh->meter_aso_en && + !(fm->transfer == attr->transfer || (!fm->ingress && !attr->ingress && attr->egress) || - (!fm->egress && !attr->egress && attr->ingress)))) + (!fm->egress && !attr->egress && attr->ingress))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Flow attributes are either invalid " @@ -6058,6 +6060,8 @@ flow_dv_mtr_alloc(struct rte_eth_dev *dev) struct mlx5_aso_mtr *mtr_free = NULL; struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; struct mlx5_aso_mtr_pool *pool; + struct rte_flow_error error; + uint8_t reg_id; uint32_t mtr_idx = 0; if (!priv->config.devx) { @@ -6080,6 +6084,20 @@ flow_dv_mtr_alloc(struct rte_eth_dev *dev) struct mlx5_aso_mtr_pool, mtrs[mtr_free->offset]); mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset); + if (!mtr_free->fm.meter_action) { + reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error); + mtr_free->fm.meter_action = + mlx5_glue->dv_create_flow_action_aso + (priv->sh->rx_domain, + pool->devx_obj->obj, + mtr_free->offset, + (1 << MLX5_FLOW_COLOR_GREEN), + reg_id - REG_C_0); + if (!mtr_free->fm.meter_action) { + flow_dv_aso_mtr_release_to_pool(dev, mtr_idx); + return 0; + } + } return mtr_idx; } @@ -11579,7 +11597,7 @@ flow_dv_translate(struct rte_eth_dev *dev, NULL, "Failed to get meter in flow."); /* Set the meter action. */ dev_flow->dv.actions[actions_n++] = - wks->fm->mfts->meter_action; + wks->fm->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12688,7 +12706,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) if (flow->meter) { fm = flow_dv_meter_find_by_idx(priv, flow->meter); if (fm) - mlx5_flow_meter_detach(fm); + mlx5_flow_meter_detach(priv, fm); flow->meter = 0; } if (flow->age) diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index bb36696172..c2a063abdb 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -602,9 +602,9 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, ebs_mantissa, val); } /* Apply modifications to meter only if it was created. */ - if (fm->mfts->meter_action) { + if (fm->meter_action) { ret = mlx5_glue->dv_modify_flow_action_meter - (fm->mfts->meter_action, &mod_attr, + (fm->meter_action, &mod_attr, rte_cpu_to_be_64(modify_bits)); if (ret) return ret; @@ -620,6 +620,7 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, (void)srtcm; (void)modify_bits; (void)active_state; + (void)is_enable; return -ENOTSUP; #endif } @@ -1405,63 +1406,91 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, { int ret = 0; - rte_spinlock_lock(&fm->sl); - if (fm->mfts->meter_action) { - if (fm->shared && - attr->transfer == fm->transfer && - attr->ingress == fm->ingress && - attr->egress == fm->egress) + if (priv->sh->meter_aso_en) { + struct mlx5_aso_mtr *aso_mtr; + + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); + if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) { + return rte_flow_error_set(error, ENOENT, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Timeout in meter configuration"); + } + rte_spinlock_lock(&fm->sl); + if (fm->shared || !fm->ref_cnt) { fm->ref_cnt++; - else + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter cannot be shared"); ret = -1; + } + rte_spinlock_unlock(&fm->sl); } else { - fm->ingress = attr->ingress; - fm->egress = attr->egress; - fm->transfer = attr->transfer; - fm->ref_cnt = 1; - /* This also creates the meter object. */ - fm->mfts->meter_action = mlx5_flow_meter_action_create(priv, - fm); - if (!fm->mfts->meter_action) { - fm->ref_cnt = 0; - fm->ingress = 0; - fm->egress = 0; - fm->transfer = 0; - ret = -1; - DRV_LOG(ERR, "Meter action create failed."); + rte_spinlock_lock(&fm->sl); + if (fm->meter_action) { + if (fm->shared && + attr->transfer == fm->transfer && + attr->ingress == fm->ingress && + attr->egress == fm->egress) { + fm->ref_cnt++; + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + fm->shared ? + "Meter attr not match." : + "Meter cannot be shared."); + ret = -1; + } + } else { + fm->ingress = attr->ingress; + fm->egress = attr->egress; + fm->transfer = attr->transfer; + fm->ref_cnt = 1; + /* This also creates the meter object. */ + fm->meter_action = mlx5_flow_meter_action_create(priv, + fm); + if (!fm->meter_action) { + fm->ref_cnt = 0; + fm->ingress = 0; + fm->egress = 0; + fm->transfer = 0; + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter action create failed."); + ret = -1; + } } + rte_spinlock_unlock(&fm->sl); } - rte_spinlock_unlock(&fm->sl); - if (ret) - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - fm->mfts->meter_action ? - "Meter attr not match" : - "Meter action create failed"); return ret ? -rte_errno : 0; } /** * Detach meter from flow. * + * @param [in] priv + * Pointer to mlx5 private data. * @param [in] fm * Pointer to flow meter. */ void -mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm) +mlx5_flow_meter_detach(struct mlx5_priv *priv, + struct mlx5_flow_meter_info *fm) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER rte_spinlock_lock(&fm->sl); MLX5_ASSERT(fm->ref_cnt); - if (--fm->ref_cnt == 0) { - mlx5_glue->destroy_flow_action(fm->mfts->meter_action); - fm->mfts->meter_action = NULL; + if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { + mlx5_glue->destroy_flow_action(fm->meter_action); + fm->meter_action = NULL; fm->ingress = 0; fm->egress = 0; fm->transfer = 0; } rte_spinlock_unlock(&fm->sl); #else + (void)priv; (void)fm; #endif } From patchwork Wed Mar 31 07:36:30 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90183 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 39AC7A034F; Wed, 31 Mar 2021 09:38:15 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 8EA13140E7C; Wed, 31 Mar 2021 09:37:02 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id B7C87140E08 for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:43 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwK001778; Wed, 31 Mar 2021 10:36:43 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:30 +0300 Message-Id: <20210331073632.1443011-13-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 12/13] net/mlx5: make ASO meter queue thread-safe X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Synchronize ASO meter queue accesses from different threads using a spinlock. Signed-off-by: Li Zhang --- drivers/net/mlx5/mlx5.h | 1 + drivers/net/mlx5/mlx5_flow_aso.c | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index d0cd402495..27a6b63ba3 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -503,6 +503,7 @@ struct mlx5_aso_sq_elem { struct mlx5_aso_sq { uint16_t log_desc_n; + rte_spinlock_t sqsl; struct mlx5_aso_cq cq; struct mlx5_devx_sq sq_obj; volatile uint64_t *uar_addr; diff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c index 4c1b2ed6e5..cdca7f9a03 100644 --- a/drivers/net/mlx5/mlx5_flow_aso.c +++ b/drivers/net/mlx5/mlx5_flow_aso.c @@ -275,6 +275,7 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, sq->tail = 0; sq->sqn = sq->sq_obj.sq->id; sq->uar_addr = mlx5_os_get_devx_uar_reg_addr(uar); + rte_spinlock_init(&sq->sqsl); return 0; error: mlx5_aso_destroy_sq(sq); @@ -666,12 +667,15 @@ mlx5_aso_mtr_sq_enqueue_single(struct mlx5_aso_sq *sq, struct mlx5_flow_meter_info *fm = NULL; uint16_t size = 1 << sq->log_desc_n; uint16_t mask = size - 1; - uint16_t res = size - (uint16_t)(sq->head - sq->tail); + uint16_t res; uint32_t dseg_idx = 0; struct mlx5_aso_mtr_pool *pool = NULL; + rte_spinlock_lock(&sq->sqsl); + res = size - (uint16_t)(sq->head - sq->tail); if (unlikely(!res)) { DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send"); + rte_spinlock_unlock(&sq->sqsl); return 0; } wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; @@ -708,6 +712,7 @@ mlx5_aso_mtr_sq_enqueue_single(struct mlx5_aso_sq *sq, rte_wmb(); *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */ rte_wmb(); + rte_spinlock_unlock(&sq->sqsl); return 1; } @@ -738,12 +743,16 @@ mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq) const unsigned int mask = cq_size - 1; uint32_t idx; uint32_t next_idx = cq->cq_ci & mask; - const uint16_t max = (uint16_t)(sq->head - sq->tail); + uint16_t max; uint16_t n = 0; int ret; - if (unlikely(!max)) + rte_spinlock_lock(&sq->sqsl); + max = (uint16_t)(sq->head - sq->tail); + if (unlikely(!max)) { + rte_spinlock_unlock(&sq->sqsl); return; + } do { idx = next_idx; next_idx = (cq->cq_ci + 1) & mask; @@ -770,6 +779,7 @@ mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq) rte_io_wmb(); cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); } + rte_spinlock_unlock(&sq->sqsl); } /** From patchwork Wed Mar 31 07:36:31 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 90180 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 7F930A034F; Wed, 31 Mar 2021 09:37:50 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3E2A6140E45; Wed, 31 Mar 2021 09:36:58 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id B5557140E07 for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:43 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwL001778; Wed, 31 Mar 2021 10:36:43 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Wed, 31 Mar 2021 10:36:31 +0300 Message-Id: <20210331073632.1443011-14-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 13/13] net/mlx5: allow multiple flow tables on the same level X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" The driver devices support creation of multiple flow tables. Jump action can be used in order to move the packet steering to different flow table. Table 0 is always the root table for packet steering. Jumping between tables may cause endless loops in steering mechanism, that's why each table has level attribute, the driver sub-system may not allow jumping to table with equal or lower level than the current table. Currently, in the driver, the table ID and level are always identical. Allow multiple flow table creation with the same level attribute. This patch adds the table id in flow table data entry, while allocates the flow table, if the table level is same but the different table id, the new table will be allocated with new table object id. It supports 4M multiple flow tables on the same level. Signed-off-by: Li Zhang --- drivers/net/mlx5/mlx5.c | 9 ++-- drivers/net/mlx5/mlx5.h | 10 ++-- drivers/net/mlx5/mlx5_flow.c | 11 +++-- drivers/net/mlx5/mlx5_flow.h | 10 ++-- drivers/net/mlx5/mlx5_flow_dv.c | 83 +++++++++++++++++++-------------- 5 files changed, 71 insertions(+), 52 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index f321dd5e18..e16a739fe4 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -1265,9 +1265,12 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) * because DV expect to see them even if they cannot be created by * RDMA-CORE. */ - if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL, 0, 1, &error) || - !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, NULL, 0, 1, &error) || - !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, NULL, 0, 1, &error)) { + if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, + NULL, 0, 1, 0, &error) || + !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, + NULL, 0, 1, 0, &error) || + !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, + NULL, 0, 1, 0, &error)) { err = ENOMEM; goto error; } diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 27a6b63ba3..1fc648d368 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -780,10 +780,12 @@ struct mlx5_aso_mtr_pools_mng { union mlx5_flow_tbl_key { struct { /* Table ID should be at the lowest address. */ - uint32_t table_id; /**< ID of the table. */ - uint16_t dummy; /**< Dummy table for DV API. */ - uint8_t domain; /**< 1 - FDB, 0 - NIC TX/RX. */ - uint8_t direction; /**< 1 - egress, 0 - ingress. */ + uint32_t level; /**< Level of the table. */ + uint32_t id:22; /**< ID of the table. */ + uint32_t dummy:1; /**< Dummy table for DV API. */ + uint32_t is_fdb:1; /**< 1 - FDB, 0 - NIC TX/RX. */ + uint32_t is_egress:1; /**< 1 - egress, 0 - ingress. */ + uint32_t reserved:7; /**< must be zero for comparison. */ }; uint64_t v64; /**< full 64bits value of key */ }; diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index f2dfe85119..d8d89bfb08 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -5500,8 +5500,7 @@ flow_create_split_sample(struct rte_eth_dev *dev, struct mlx5_flow_tbl_data_entry, tbl); sfx_attr.group = sfx_attr.transfer ? - (sfx_tbl_data->table_id - 1) : - sfx_tbl_data->table_id; + (sfx_tbl_data->level - 1) : sfx_tbl_data->level; } else { MLX5_ASSERT(attr->transfer); sfx_attr.group = jump_table; @@ -7754,10 +7753,12 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) union tunnel_offload_mark mbits = { .val = mark }; union mlx5_flow_tbl_key table_key = { { - .table_id = tunnel_id_to_flow_tbl(mbits.table_id), + .level = tunnel_id_to_flow_tbl(mbits.table_id), + .id = 0, + .reserved = 0, .dummy = 0, - .domain = !!mbits.transfer, - .direction = 0, + .is_fdb = !!mbits.transfer, + .is_egress = 0, } }; he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 9e146685c2..59e9ce2c9e 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -565,8 +565,9 @@ struct mlx5_flow_tbl_data_entry { uint32_t is_egress:1; /**< Egress table. */ uint32_t is_transfer:1; /**< Transfer table. */ uint32_t dummy:1; /**< DR table. */ - uint32_t reserve:27; /**< Reserved to future using. */ - uint32_t table_id; /**< Table ID. */ + uint32_t id:22; /**< Table ID. */ + uint32_t reserve:5; /**< Reserved to future using. */ + uint32_t level; /**< Table level. */ }; /* Sub rdma-core actions list. */ @@ -1393,9 +1394,10 @@ int flow_dv_tbl_match_cb(struct mlx5_hlist *list, void flow_dv_tbl_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry); struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev, - uint32_t table_id, uint8_t egress, uint8_t transfer, + uint32_t table_level, uint8_t egress, uint8_t transfer, bool external, const struct mlx5_flow_tunnel *tunnel, - uint32_t group_id, uint8_t dummy, struct rte_flow_error *error); + uint32_t group_id, uint8_t dummy, + uint32_t table_id, struct rte_flow_error *error); struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *cb_ctx); diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 58bd279240..c4c54d4043 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -9342,20 +9342,21 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) tbl_data->group_id = tt_prm->group_id; tbl_data->external = !!tt_prm->external; tbl_data->tunnel_offload = is_tunnel_offload_active(dev); - tbl_data->is_egress = !!key.direction; - tbl_data->is_transfer = !!key.domain; + tbl_data->is_egress = !!key.is_egress; + tbl_data->is_transfer = !!key.is_fdb; tbl_data->dummy = !!key.dummy; - tbl_data->table_id = key.table_id; + tbl_data->level = key.level; + tbl_data->id = key.id; tbl = &tbl_data->tbl; if (key.dummy) return &tbl_data->entry; - if (key.domain) + if (key.is_fdb) domain = sh->fdb_domain; - else if (key.direction) + else if (key.is_egress) domain = sh->tx_domain; else domain = sh->rx_domain; - ret = mlx5_flow_os_create_flow_tbl(domain, key.table_id, &tbl->obj); + ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj); if (ret) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -9363,7 +9364,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); return NULL; } - if (key.table_id) { + if (key.level != 0) { ret = mlx5_flow_os_create_flow_action_dest_flow_tbl (tbl->obj, &tbl_data->jump.action); if (ret) { @@ -9376,9 +9377,9 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) return NULL; } } - MKSTR(matcher_name, "%s_%s_%u_matcher_cache", - key.domain ? "FDB" : "NIC", key.direction ? "egress" : "ingress", - key.table_id); + MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache", + key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress", + key.level, key.id); mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh, flow_dv_matcher_create_cb, flow_dv_matcher_match_cb, @@ -9395,10 +9396,11 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused, container_of(entry, struct mlx5_flow_tbl_data_entry, entry); union mlx5_flow_tbl_key key = { .v64 = key64 }; - return tbl_data->table_id != key.table_id || + return tbl_data->level != key.level || + tbl_data->id != key.id || tbl_data->dummy != key.dummy || - tbl_data->is_transfer != key.domain || - tbl_data->is_egress != key.direction; + tbl_data->is_transfer != !!key.is_fdb || + tbl_data->is_egress != !!key.is_egress; } /** @@ -9406,14 +9408,16 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused, * * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in] table_id - * Table id to use. + * @param[in] table_level + * Table level to use. * @param[in] egress * Direction of the table. * @param[in] transfer * E-Switch or NIC flow. * @param[in] dummy * Dummy entry for dv API. + * @param[in] table_id + * Table id to use. * @param[out] error * pointer to error structure. * @@ -9422,20 +9426,23 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused, */ struct mlx5_flow_tbl_resource * flow_dv_tbl_resource_get(struct rte_eth_dev *dev, - uint32_t table_id, uint8_t egress, + uint32_t table_level, uint8_t egress, uint8_t transfer, bool external, const struct mlx5_flow_tunnel *tunnel, uint32_t group_id, uint8_t dummy, + uint32_t table_id, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; union mlx5_flow_tbl_key table_key = { { - .table_id = table_id, - .dummy = dummy, - .domain = !!transfer, - .direction = !!egress, + .level = table_level, + .id = table_id, + .reserved = 0, + .dummy = !!dummy, + .is_fdb = !!transfer, + .is_egress = !!egress, } }; struct mlx5_flow_tbl_tunnel_prm tt_prm = { @@ -9458,8 +9465,10 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, "cannot get table"); return NULL; } - DRV_LOG(DEBUG, "Table_id %u tunnel %u group %u registered.", - table_id, tunnel ? tunnel->tunnel_id : 0, group_id); + DRV_LOG(DEBUG, "table_level %u table_id %u " + "tunnel %u group %u registered.", + table_level, table_id, + tunnel ? tunnel->tunnel_id : 0, group_id); tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry); return &tbl_data->tbl; } @@ -9486,7 +9495,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list, tbl_data->tunnel->tunnel_id : 0, .group = tbl_data->group_id }; - uint32_t table_id = tbl_data->table_id; + uint32_t table_level = tbl_data->level; tunnel_grp_hash = tbl_data->tunnel ? tbl_data->tunnel->groups : @@ -9495,8 +9504,9 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list, if (he) mlx5_hlist_unregister(tunnel_grp_hash, he); DRV_LOG(DEBUG, - "Table_id %u tunnel %u group %u released.", - table_id, + "table_level %u id %u tunnel %u group %u released.", + table_level, + tbl_data->id, tbl_data->tunnel ? tbl_data->tunnel->tunnel_id : 0, tbl_data->group_id); @@ -9624,10 +9634,10 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, * tunnel offload API requires this registration for cases when * tunnel match rule was inserted before tunnel set rule. */ - tbl = flow_dv_tbl_resource_get(dev, key->table_id, - key->direction, key->domain, + tbl = flow_dv_tbl_resource_get(dev, key->level, + key->is_egress, key->is_fdb, dev_flow->external, tunnel, - group_id, 0, error); + group_id, 0, 0, error); if (!tbl) return -rte_errno; /* No need to refill the error info */ tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); @@ -10106,7 +10116,7 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused, is_egress = 1; tbl = flow_dv_tbl_resource_get(dev, next_ft_id, is_egress, is_transfer, - true, NULL, 0, 0, error); + true, NULL, 0, 0, 0, error); if (!tbl) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -11471,7 +11481,7 @@ flow_dv_translate(struct rte_eth_dev *dev, attr->transfer, !!dev_flow->external, tunnel, jump_group, 0, - error); + 0, error); if (!tbl) return rte_flow_error_set (error, errno, @@ -12013,9 +12023,9 @@ flow_dv_translate(struct rte_eth_dev *dev, matcher.priority = mlx5_get_matcher_priority(dev, attr, matcher.priority); /* reserved field no needs to be set to 0 here. */ - tbl_key.domain = attr->transfer; - tbl_key.direction = attr->egress; - tbl_key.table_id = dev_flow->dv.group; + tbl_key.is_fdb = attr->transfer; + tbl_key.is_egress = attr->egress; + tbl_key.level = dev_flow->dv.group; if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, tunnel, attr->group, error)) return -rte_errno; @@ -13486,7 +13496,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, /* Create the meter table with METER level. */ dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, egress, transfer, false, NULL, 0, - 0, &error); + 0, 0, &error); if (!dtb->tbl) { DRV_LOG(ERR, "Failed to create meter policer table."); return -1; @@ -13495,7 +13505,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, dtb->sfx_tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_SUFFIX, egress, transfer, false, NULL, 0, - 0, &error); + 0, 0, &error); if (!dtb->sfx_tbl) { DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; @@ -14018,7 +14028,8 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) void *flow = NULL; int ret = -1; - tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, 0, NULL); + tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, + 0, 0, 0, NULL); if (!tbl) goto err; dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);