From patchwork Tue Apr 20 10:55:17 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jiawei Wang X-Patchwork-Id: 91873 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8DA01A0548; Tue, 20 Apr 2021 12:56:51 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 4D0A941798; Tue, 20 Apr 2021 12:55:46 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 9C3BC41745 for ; Tue, 20 Apr 2021 12:55:30 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from jiaweiw@nvidia.com) with SMTP; 20 Apr 2021 13:55:25 +0300 Received: from nvidia.com (gen-l-vrt-281.mtl.labs.mlnx [10.237.44.1]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 13KAtMRw009943; Tue, 20 Apr 2021 13:55:25 +0300 From: Jiawei Wang To: matan@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, ferruh.yigit@intel.com, thomas@monjalon.net, Shahaf Shuler Cc: dev@dpdk.org, rasland@nvidia.com, asafp@nvidia.com, Li Zhang Date: Tue, 20 Apr 2021 13:55:17 +0300 Message-Id: <1618916122-181792-11-git-send-email-jiaweiw@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com> Subject: [dpdk-dev] [PATCH v6 10/15] net/mlx5: initialize the flow meter ASO SQ X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Li Zhang Initialize the flow meter ASO SQ WQEs with all the constant data that should not be updated per enqueue operation. Signed-off-by: Li Zhang Acked-by: Matan Azrad --- drivers/net/mlx5/linux/mlx5_os.c | 16 + drivers/net/mlx5/meson.build | 2 +- drivers/net/mlx5/mlx5.c | 68 +++- drivers/net/mlx5/mlx5.h | 22 +- drivers/net/mlx5/mlx5_flow.h | 4 +- drivers/net/mlx5/mlx5_flow_age.c | 591 --------------------------------- drivers/net/mlx5/mlx5_flow_aso.c | 659 +++++++++++++++++++++++++++++++++++++ drivers/net/mlx5/mlx5_flow_dv.c | 7 +- drivers/net/mlx5/mlx5_flow_meter.c | 7 +- 9 files changed, 767 insertions(+), 609 deletions(-) delete mode 100644 drivers/net/mlx5/mlx5_flow_age.c create mode 100644 drivers/net/mlx5/mlx5_flow_aso.c diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index ad43141..336cdbe 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -1290,6 +1290,22 @@ priv->mtr_color_reg); } } + if (config->hca_attr.qos.sup && + config->hca_attr.qos.flow_meter_aso_sup) { + uint32_t log_obj_size = + rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); + if (log_obj_size >= + config->hca_attr.qos.log_meter_aso_granularity && + log_obj_size <= + config->hca_attr.qos.log_meter_aso_max_alloc) { + sh->meter_aso_en = 1; + err = mlx5_aso_flow_mtrs_mng_init(priv); + if (err) { + err = -err; + goto error; + } + } + } #endif #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO if (config->hca_attr.flow_hit_aso && diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build index 8740ca5..0ec3002 100644 --- a/drivers/net/mlx5/meson.build +++ b/drivers/net/mlx5/meson.build @@ -15,7 +15,7 @@ sources = files( 'mlx5_flow.c', 'mlx5_flow_meter.c', 'mlx5_flow_dv.c', - 'mlx5_flow_age.c', + 'mlx5_flow_aso.c', 'mlx5_mac.c', 'mlx5_mr.c', 'mlx5_rss.c', diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 1b5b5cb..00055e3 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -403,7 +403,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = rte_errno = ENOMEM; return -ENOMEM; } - err = mlx5_aso_queue_init(sh); + err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT); if (err) { mlx5_free(sh->aso_age_mng); return -1; @@ -425,8 +425,8 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = { int i, j; - mlx5_aso_queue_stop(sh); - mlx5_aso_queue_uninit(sh); + mlx5_aso_flow_hit_queue_poll_stop(sh); + mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); if (sh->aso_age_mng->pools) { struct mlx5_aso_age_pool *pool; @@ -564,6 +564,66 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = memset(&sh->cmng, 0, sizeof(sh->cmng)); } +/** + * Initialize the aso flow meters management structure. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object to free + */ +int +mlx5_aso_flow_mtrs_mng_init(struct mlx5_priv *priv) +{ + if (!priv->mtr_idx_tbl) { + priv->mtr_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); + if (!priv->mtr_idx_tbl) { + DRV_LOG(ERR, "fail to create meter lookup table."); + rte_errno = ENOMEM; + return -ENOMEM; + } + } + if (!priv->sh->mtrmng) { + priv->sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*priv->sh->mtrmng), + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); + if (!priv->sh->mtrmng) { + DRV_LOG(ERR, "mlx5_aso_mtr_pools_mng allocation was failed."); + rte_errno = ENOMEM; + return -ENOMEM; + } + rte_spinlock_init(&priv->sh->mtrmng->mtrsl); + LIST_INIT(&priv->sh->mtrmng->meters); + } + return 0; +} + +/** + * Close and release all the resources of + * the ASO flow meter management structure. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object to free. + */ +static void +mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) +{ + struct mlx5_aso_mtr_pool *mtr_pool; + struct mlx5_aso_mtr_pools_mng *mtrmng = sh->mtrmng; + uint32_t idx; + + mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); + idx = mtrmng->n_valid; + while (idx--) { + mtr_pool = mtrmng->pools[idx]; + claim_zero(mlx5_devx_cmd_destroy + (mtr_pool->devx_obj)); + mtrmng->n_valid--; + mlx5_free(mtr_pool); + } + mlx5_free(sh->mtrmng->pools); + mlx5_free(sh->mtrmng); + sh->mtrmng = NULL; +} + /* Send FLOW_AGED event if needed. */ void mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) @@ -1113,6 +1173,8 @@ struct mlx5_dev_ctx_shared * mlx5_flow_aso_age_mng_close(sh); sh->aso_age_mng = NULL; } + if (sh->mtrmng) + mlx5_aso_flow_mtrs_mng_close(sh); mlx5_flow_ipool_destroy(sh); mlx5_os_dev_shared_handler_uninstall(sh); if (sh->cnt_id_tbl) { diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 2e93dda..4ad0e14 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -491,8 +491,13 @@ struct mlx5_aso_devx_mr { }; struct mlx5_aso_sq_elem { - struct mlx5_aso_age_pool *pool; - uint16_t burst_size; + union { + struct { + struct mlx5_aso_age_pool *pool; + uint16_t burst_size; + }; + struct mlx5_aso_mtr *mtr; + }; }; struct mlx5_aso_sq { @@ -764,7 +769,6 @@ struct mlx5_aso_mtr_pools_mng { volatile uint16_t n_valid; /* Number of valid pools. */ uint16_t n; /* Number of pools. */ rte_spinlock_t mtrsl; /* The ASO flow meter free list lock. */ - struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */ struct aso_meter_list meters; /* Free ASO flow meter list. */ struct mlx5_aso_sq sq; /*SQ using by ASO flow meter. */ struct mlx5_aso_mtr_pool **pools; /* ASO flow meter pool array. */ @@ -1195,6 +1199,7 @@ struct mlx5_priv { uint8_t mtr_color_reg; /* Meter color match REG_C. */ struct mlx5_mtr_profiles flow_meter_profiles; /* MTR profile list. */ struct mlx5_legacy_flow_meters flow_meters; /* MTR list. */ + struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */ uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */ uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */ struct mlx5_mp_id mp_id; /* ID of a multi-process process */ @@ -1258,6 +1263,7 @@ int mlx5_hairpin_cap_get(struct rte_eth_dev *dev, bool mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev); int mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev); int mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_flow_mtrs_mng_init(struct mlx5_priv *priv); /* mlx5_ethdev.c */ @@ -1528,9 +1534,11 @@ int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev, /* mlx5_flow_aso.c */ -int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh); -int mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh); -int mlx5_aso_queue_stop(struct mlx5_dev_ctx_shared *sh); -void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod); +int mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh); +void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod); #endif /* RTE_PMD_MLX5_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index b6ec727..81cf1b5 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -826,8 +826,8 @@ struct mlx5_flow { #define MLX5_FLOW_METER_DISABLE 0 #define MLX5_FLOW_METER_ENABLE 1 -#define MLX5_ASO_CQE_RESPONSE_DELAY 10 -#define MLX5_MTR_POLL_CQE_TIMES 100000u +#define MLX5_ASO_WQE_CQE_RESPONSE_DELAY 10u +#define MLX5_MTR_POLL_WQE_CQE_TIMES 100000u #define MLX5_MAN_WIDTH 8 /* Legacy Meter parameter structure. */ diff --git a/drivers/net/mlx5/mlx5_flow_age.c b/drivers/net/mlx5/mlx5_flow_age.c deleted file mode 100644 index 00cb20d..0000000 --- a/drivers/net/mlx5/mlx5_flow_age.c +++ /dev/null @@ -1,591 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright 2020 Mellanox Technologies, Ltd - */ -#include -#include -#include -#include - -#include -#include -#include - -#include "mlx5.h" -#include "mlx5_flow.h" - - -/** - * Destroy Completion Queue used for ASO access. - * - * @param[in] cq - * ASO CQ to destroy. - */ -static void -mlx5_aso_cq_destroy(struct mlx5_aso_cq *cq) -{ - if (cq->cq_obj.cq) - mlx5_devx_cq_destroy(&cq->cq_obj); - memset(cq, 0, sizeof(*cq)); -} - -/** - * Create Completion Queue used for ASO access. - * - * @param[in] ctx - * Context returned from mlx5 open_device() glue function. - * @param[in/out] cq - * Pointer to CQ to create. - * @param[in] log_desc_n - * Log of number of descriptors in queue. - * @param[in] socket - * Socket to use for allocation. - * @param[in] uar_page_id - * UAR page ID to use. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -static int -mlx5_aso_cq_create(void *ctx, struct mlx5_aso_cq *cq, uint16_t log_desc_n, - int socket, int uar_page_id) -{ - struct mlx5_devx_cq_attr attr = { - .uar_page_id = uar_page_id, - }; - - cq->log_desc_n = log_desc_n; - cq->cq_ci = 0; - return mlx5_devx_cq_create(ctx, &cq->cq_obj, log_desc_n, &attr, socket); -} - -/** - * Free MR resources. - * - * @param[in] mr - * MR to free. - */ -static void -mlx5_aso_devx_dereg_mr(struct mlx5_aso_devx_mr *mr) -{ - claim_zero(mlx5_devx_cmd_destroy(mr->mkey)); - if (!mr->is_indirect && mr->umem) - claim_zero(mlx5_glue->devx_umem_dereg(mr->umem)); - mlx5_free(mr->buf); - memset(mr, 0, sizeof(*mr)); -} - -/** - * Register Memory Region. - * - * @param[in] ctx - * Context returned from mlx5 open_device() glue function. - * @param[in] length - * Size of MR buffer. - * @param[in/out] mr - * Pointer to MR to create. - * @param[in] socket - * Socket to use for allocation. - * @param[in] pdn - * Protection Domain number to use. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -static int -mlx5_aso_devx_reg_mr(void *ctx, size_t length, struct mlx5_aso_devx_mr *mr, - int socket, int pdn) -{ - struct mlx5_devx_mkey_attr mkey_attr; - - mr->buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096, - socket); - if (!mr->buf) { - DRV_LOG(ERR, "Failed to create ASO bits mem for MR by Devx."); - return -1; - } - mr->umem = mlx5_os_umem_reg(ctx, mr->buf, length, - IBV_ACCESS_LOCAL_WRITE); - if (!mr->umem) { - DRV_LOG(ERR, "Failed to register Umem for MR by Devx."); - goto error; - } - mkey_attr.addr = (uintptr_t)mr->buf; - mkey_attr.size = length; - mkey_attr.umem_id = mlx5_os_get_umem_id(mr->umem); - mkey_attr.pd = pdn; - mkey_attr.pg_access = 1; - mkey_attr.klm_array = NULL; - mkey_attr.klm_num = 0; - mkey_attr.relaxed_ordering_read = 0; - mkey_attr.relaxed_ordering_write = 0; - mr->mkey = mlx5_devx_cmd_mkey_create(ctx, &mkey_attr); - if (!mr->mkey) { - DRV_LOG(ERR, "Failed to create direct Mkey."); - goto error; - } - mr->length = length; - mr->is_indirect = false; - return 0; -error: - if (mr->umem) - claim_zero(mlx5_glue->devx_umem_dereg(mr->umem)); - mlx5_free(mr->buf); - return -1; -} - -/** - * Destroy Send Queue used for ASO access. - * - * @param[in] sq - * ASO SQ to destroy. - */ -static void -mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq) -{ - mlx5_devx_sq_destroy(&sq->sq_obj); - mlx5_aso_cq_destroy(&sq->cq); - mlx5_aso_devx_dereg_mr(&sq->mr); - memset(sq, 0, sizeof(*sq)); -} - -/** - * Initialize Send Queue used for ASO access. - * - * @param[in] sq - * ASO SQ to initialize. - */ -static void -mlx5_aso_init_sq(struct mlx5_aso_sq *sq) -{ - volatile struct mlx5_aso_wqe *restrict wqe; - int i; - int size = 1 << sq->log_desc_n; - uint64_t addr; - - /* All the next fields state should stay constant. */ - for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) { - wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | - (sizeof(*wqe) >> 4)); - wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.mkey->id); - addr = (uint64_t)((uint64_t *)sq->mr.buf + i * - MLX5_ASO_AGE_ACTIONS_PER_POOL / 64); - wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32)); - wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u); - wqe->aso_cseg.operand_masks = rte_cpu_to_be_32 - (0u | - (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | - (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) | - (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) | - (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); - wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX); - } -} - -/** - * Create Send Queue used for ASO access. - * - * @param[in] ctx - * Context returned from mlx5 open_device() glue function. - * @param[in/out] sq - * Pointer to SQ to create. - * @param[in] socket - * Socket to use for allocation. - * @param[in] uar - * User Access Region object. - * @param[in] pdn - * Protection Domain number to use. - * @param[in] log_desc_n - * Log of number of descriptors in queue. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -static int -mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, - void *uar, uint32_t pdn, uint16_t log_desc_n, - uint32_t ts_format) -{ - struct mlx5_devx_create_sq_attr attr = { - .user_index = 0xFFFF, - .wq_attr = (struct mlx5_devx_wq_attr){ - .pd = pdn, - .uar_page = mlx5_os_get_devx_uar_page_id(uar), - }, - .ts_format = mlx5_ts_format_conv(ts_format), - }; - struct mlx5_devx_modify_sq_attr modify_attr = { - .state = MLX5_SQC_STATE_RDY, - }; - uint32_t sq_desc_n = 1 << log_desc_n; - uint16_t log_wqbb_n; - int ret; - - if (mlx5_aso_devx_reg_mr(ctx, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * - sq_desc_n, &sq->mr, socket, pdn)) - return -1; - if (mlx5_aso_cq_create(ctx, &sq->cq, log_desc_n, socket, - mlx5_os_get_devx_uar_page_id(uar))) - goto error; - sq->log_desc_n = log_desc_n; - attr.cqn = sq->cq.cq_obj.cq->id; - /* for mlx5_aso_wqe that is twice the size of mlx5_wqe */ - log_wqbb_n = log_desc_n + 1; - ret = mlx5_devx_sq_create(ctx, &sq->sq_obj, log_wqbb_n, &attr, socket); - if (ret) { - DRV_LOG(ERR, "Can't create SQ object."); - rte_errno = ENOMEM; - goto error; - } - ret = mlx5_devx_cmd_modify_sq(sq->sq_obj.sq, &modify_attr); - if (ret) { - DRV_LOG(ERR, "Can't change SQ state to ready."); - rte_errno = ENOMEM; - goto error; - } - sq->pi = 0; - sq->head = 0; - sq->tail = 0; - sq->sqn = sq->sq_obj.sq->id; - sq->uar_addr = mlx5_os_get_devx_uar_reg_addr(uar); - mlx5_aso_init_sq(sq); - return 0; -error: - mlx5_aso_destroy_sq(sq); - return -1; -} - -/** - * API to create and initialize Send Queue used for ASO access. - * - * @param[in] sh - * Pointer to shared device context. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -int -mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh) -{ - return mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0, - sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC, - sh->sq_ts_format); -} - -/** - * API to destroy Send Queue used for ASO access. - * - * @param[in] sh - * Pointer to shared device context. - */ -void -mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh) -{ - mlx5_aso_destroy_sq(&sh->aso_age_mng->aso_sq); -} - -/** - * Write a burst of WQEs to ASO SQ. - * - * @param[in] mng - * ASO management data, contains the SQ. - * @param[in] n - * Index of the last valid pool. - * - * @return - * Number of WQEs in burst. - */ -static uint16_t -mlx5_aso_sq_enqueue_burst(struct mlx5_aso_age_mng *mng, uint16_t n) -{ - volatile struct mlx5_aso_wqe *wqe; - struct mlx5_aso_sq *sq = &mng->aso_sq; - struct mlx5_aso_age_pool *pool; - uint16_t size = 1 << sq->log_desc_n; - uint16_t mask = size - 1; - uint16_t max; - uint16_t start_head = sq->head; - - max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), n - sq->next); - if (unlikely(!max)) - return 0; - sq->elts[start_head & mask].burst_size = max; - do { - wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; - rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); - /* Fill next WQE. */ - rte_spinlock_lock(&mng->resize_sl); - pool = mng->pools[sq->next]; - rte_spinlock_unlock(&mng->resize_sl); - sq->elts[sq->head & mask].pool = pool; - wqe->general_cseg.misc = - rte_cpu_to_be_32(((struct mlx5_devx_obj *) - (pool->flow_hit_aso_obj))->id); - wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << - MLX5_COMP_MODE_OFFSET); - wqe->general_cseg.opcode = rte_cpu_to_be_32 - (MLX5_OPCODE_ACCESS_ASO | - (ASO_OPC_MOD_FLOW_HIT << - WQE_CSEG_OPC_MOD_OFFSET) | - (sq->pi << - WQE_CSEG_WQE_INDEX_OFFSET)); - sq->pi += 2; /* Each WQE contains 2 WQEBB's. */ - sq->head++; - sq->next++; - max--; - } while (max); - wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << - MLX5_COMP_MODE_OFFSET); - rte_io_wmb(); - sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi); - rte_wmb(); - *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH.*/ - rte_wmb(); - return sq->elts[start_head & mask].burst_size; -} - -/** - * Debug utility function. Dump contents of error CQE and WQE. - * - * @param[in] cqe - * Error CQE to dump. - * @param[in] wqe - * Error WQE to dump. - */ -static void -mlx5_aso_dump_err_objs(volatile uint32_t *cqe, volatile uint32_t *wqe) -{ - int i; - - DRV_LOG(ERR, "Error cqe:"); - for (i = 0; i < 16; i += 4) - DRV_LOG(ERR, "%08X %08X %08X %08X", cqe[i], cqe[i + 1], - cqe[i + 2], cqe[i + 3]); - DRV_LOG(ERR, "\nError wqe:"); - for (i = 0; i < (int)sizeof(struct mlx5_aso_wqe) / 4; i += 4) - DRV_LOG(ERR, "%08X %08X %08X %08X", wqe[i], wqe[i + 1], - wqe[i + 2], wqe[i + 3]); -} - -/** - * Handle case of error CQE. - * - * @param[in] sq - * ASO SQ to use. - */ -static void -mlx5_aso_cqe_err_handle(struct mlx5_aso_sq *sq) -{ - struct mlx5_aso_cq *cq = &sq->cq; - uint32_t idx = cq->cq_ci & ((1 << cq->log_desc_n) - 1); - volatile struct mlx5_err_cqe *cqe = - (volatile struct mlx5_err_cqe *)&cq->cq_obj.cqes[idx]; - - cq->errors++; - idx = rte_be_to_cpu_16(cqe->wqe_counter) & (1u << sq->log_desc_n); - mlx5_aso_dump_err_objs((volatile uint32_t *)cqe, - (volatile uint32_t *)&sq->sq_obj.aso_wqes[idx]); -} - -/** - * Update ASO objects upon completion. - * - * @param[in] sh - * Shared device context. - * @param[in] n - * Number of completed ASO objects. - */ -static void -mlx5_aso_age_action_update(struct mlx5_dev_ctx_shared *sh, uint16_t n) -{ - struct mlx5_aso_age_mng *mng = sh->aso_age_mng; - struct mlx5_aso_sq *sq = &mng->aso_sq; - struct mlx5_age_info *age_info; - const uint16_t size = 1 << sq->log_desc_n; - const uint16_t mask = size - 1; - const uint64_t curr = MLX5_CURR_TIME_SEC; - uint16_t expected = AGE_CANDIDATE; - uint16_t i; - - for (i = 0; i < n; ++i) { - uint16_t idx = (sq->tail + i) & mask; - struct mlx5_aso_age_pool *pool = sq->elts[idx].pool; - uint64_t diff = curr - pool->time_of_last_age_check; - uint64_t *addr = sq->mr.buf; - int j; - - addr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64; - pool->time_of_last_age_check = curr; - for (j = 0; j < MLX5_ASO_AGE_ACTIONS_PER_POOL; j++) { - struct mlx5_aso_age_action *act = &pool->actions[j]; - struct mlx5_age_param *ap = &act->age_params; - uint8_t byte; - uint8_t offset; - uint8_t *u8addr; - uint8_t hit; - - if (__atomic_load_n(&ap->state, __ATOMIC_RELAXED) != - AGE_CANDIDATE) - continue; - byte = 63 - (j / 8); - offset = j % 8; - u8addr = (uint8_t *)addr; - hit = (u8addr[byte] >> offset) & 0x1; - if (hit) { - __atomic_store_n(&ap->sec_since_last_hit, 0, - __ATOMIC_RELAXED); - } else { - struct mlx5_priv *priv; - - __atomic_fetch_add(&ap->sec_since_last_hit, - diff, __ATOMIC_RELAXED); - /* If timeout passed add to aged-out list. */ - if (ap->sec_since_last_hit <= ap->timeout) - continue; - priv = - rte_eth_devices[ap->port_id].data->dev_private; - age_info = GET_PORT_AGE_INFO(priv); - rte_spinlock_lock(&age_info->aged_sl); - if (__atomic_compare_exchange_n(&ap->state, - &expected, - AGE_TMOUT, - false, - __ATOMIC_RELAXED, - __ATOMIC_RELAXED)) { - LIST_INSERT_HEAD(&age_info->aged_aso, - act, next); - MLX5_AGE_SET(age_info, - MLX5_AGE_EVENT_NEW); - } - rte_spinlock_unlock(&age_info->aged_sl); - } - } - } - mlx5_age_event_prepare(sh); -} - -/** - * Handle completions from WQEs sent to ASO SQ. - * - * @param[in] sh - * Shared device context. - * - * @return - * Number of CQEs handled. - */ -static uint16_t -mlx5_aso_completion_handle(struct mlx5_dev_ctx_shared *sh) -{ - struct mlx5_aso_age_mng *mng = sh->aso_age_mng; - struct mlx5_aso_sq *sq = &mng->aso_sq; - struct mlx5_aso_cq *cq = &sq->cq; - volatile struct mlx5_cqe *restrict cqe; - const unsigned int cq_size = 1 << cq->log_desc_n; - const unsigned int mask = cq_size - 1; - uint32_t idx; - uint32_t next_idx = cq->cq_ci & mask; - const uint16_t max = (uint16_t)(sq->head - sq->tail); - uint16_t i = 0; - int ret; - if (unlikely(!max)) - return 0; - do { - idx = next_idx; - next_idx = (cq->cq_ci + 1) & mask; - rte_prefetch0(&cq->cq_obj.cqes[next_idx]); - cqe = &cq->cq_obj.cqes[idx]; - ret = check_cqe(cqe, cq_size, cq->cq_ci); - /* - * Be sure owner read is done before any other cookie field or - * opaque field. - */ - rte_io_rmb(); - if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { - if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) - break; - mlx5_aso_cqe_err_handle(sq); - } else { - i += sq->elts[(sq->tail + i) & mask].burst_size; - } - cq->cq_ci++; - } while (1); - if (likely(i)) { - mlx5_aso_age_action_update(sh, i); - sq->tail += i; - rte_io_wmb(); - cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); - } - return i; -} - -/** - * Periodically read CQEs and send WQEs to ASO SQ. - * - * @param[in] arg - * Shared device context containing the ASO SQ. - */ -static void -mlx5_flow_aso_alarm(void *arg) -{ - struct mlx5_dev_ctx_shared *sh = arg; - struct mlx5_aso_sq *sq = &sh->aso_age_mng->aso_sq; - uint32_t us = 100u; - uint16_t n; - - rte_spinlock_lock(&sh->aso_age_mng->resize_sl); - n = sh->aso_age_mng->next; - rte_spinlock_unlock(&sh->aso_age_mng->resize_sl); - mlx5_aso_completion_handle(sh); - if (sq->next == n) { - /* End of loop: wait 1 second. */ - us = US_PER_S; - sq->next = 0; - } - mlx5_aso_sq_enqueue_burst(sh->aso_age_mng, n); - if (rte_eal_alarm_set(us, mlx5_flow_aso_alarm, sh)) - DRV_LOG(ERR, "Cannot reinitialize aso alarm."); -} - -/** - * API to start ASO access using ASO SQ. - * - * @param[in] sh - * Pointer to shared device context. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -int -mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh) -{ - if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) { - DRV_LOG(ERR, "Cannot reinitialize ASO age alarm."); - return -rte_errno; - } - return 0; -} - -/** - * API to stop ASO access using ASO SQ. - * - * @param[in] sh - * Pointer to shared device context. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -int -mlx5_aso_queue_stop(struct mlx5_dev_ctx_shared *sh) -{ - int retries = 1024; - - if (!sh->aso_age_mng->aso_sq.sq_obj.sq) - return -EINVAL; - rte_errno = 0; - while (--retries) { - rte_eal_alarm_cancel(mlx5_flow_aso_alarm, sh); - if (rte_errno != EINPROGRESS) - break; - rte_pause(); - } - return -rte_errno; -} diff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c new file mode 100644 index 0000000..067471b --- /dev/null +++ b/drivers/net/mlx5/mlx5_flow_aso.c @@ -0,0 +1,659 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2020 Mellanox Technologies, Ltd + */ +#include +#include +#include +#include + +#include +#include +#include + +#include "mlx5.h" +#include "mlx5_flow.h" + + +/** + * Destroy Completion Queue used for ASO access. + * + * @param[in] cq + * ASO CQ to destroy. + */ +static void +mlx5_aso_cq_destroy(struct mlx5_aso_cq *cq) +{ + if (cq->cq_obj.cq) + mlx5_devx_cq_destroy(&cq->cq_obj); + memset(cq, 0, sizeof(*cq)); +} + +/** + * Create Completion Queue used for ASO access. + * + * @param[in] ctx + * Context returned from mlx5 open_device() glue function. + * @param[in/out] cq + * Pointer to CQ to create. + * @param[in] log_desc_n + * Log of number of descriptors in queue. + * @param[in] socket + * Socket to use for allocation. + * @param[in] uar_page_id + * UAR page ID to use. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_aso_cq_create(void *ctx, struct mlx5_aso_cq *cq, uint16_t log_desc_n, + int socket, int uar_page_id) +{ + struct mlx5_devx_cq_attr attr = { + .uar_page_id = uar_page_id, + }; + + cq->log_desc_n = log_desc_n; + cq->cq_ci = 0; + return mlx5_devx_cq_create(ctx, &cq->cq_obj, log_desc_n, &attr, socket); +} + +/** + * Free MR resources. + * + * @param[in] mr + * MR to free. + */ +static void +mlx5_aso_devx_dereg_mr(struct mlx5_aso_devx_mr *mr) +{ + claim_zero(mlx5_devx_cmd_destroy(mr->mkey)); + if (!mr->is_indirect && mr->umem) + claim_zero(mlx5_glue->devx_umem_dereg(mr->umem)); + mlx5_free(mr->buf); + memset(mr, 0, sizeof(*mr)); +} + +/** + * Register Memory Region. + * + * @param[in] ctx + * Context returned from mlx5 open_device() glue function. + * @param[in] length + * Size of MR buffer. + * @param[in/out] mr + * Pointer to MR to create. + * @param[in] socket + * Socket to use for allocation. + * @param[in] pdn + * Protection Domain number to use. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_aso_devx_reg_mr(void *ctx, size_t length, struct mlx5_aso_devx_mr *mr, + int socket, int pdn) +{ + struct mlx5_devx_mkey_attr mkey_attr; + + mr->buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096, + socket); + if (!mr->buf) { + DRV_LOG(ERR, "Failed to create ASO bits mem for MR by Devx."); + return -1; + } + mr->umem = mlx5_os_umem_reg(ctx, mr->buf, length, + IBV_ACCESS_LOCAL_WRITE); + if (!mr->umem) { + DRV_LOG(ERR, "Failed to register Umem for MR by Devx."); + goto error; + } + mkey_attr.addr = (uintptr_t)mr->buf; + mkey_attr.size = length; + mkey_attr.umem_id = mlx5_os_get_umem_id(mr->umem); + mkey_attr.pd = pdn; + mkey_attr.pg_access = 1; + mkey_attr.klm_array = NULL; + mkey_attr.klm_num = 0; + mkey_attr.relaxed_ordering_read = 0; + mkey_attr.relaxed_ordering_write = 0; + mr->mkey = mlx5_devx_cmd_mkey_create(ctx, &mkey_attr); + if (!mr->mkey) { + DRV_LOG(ERR, "Failed to create direct Mkey."); + goto error; + } + mr->length = length; + mr->is_indirect = false; + return 0; +error: + if (mr->umem) + claim_zero(mlx5_glue->devx_umem_dereg(mr->umem)); + mlx5_free(mr->buf); + return -1; +} + +/** + * Destroy Send Queue used for ASO access. + * + * @param[in] sq + * ASO SQ to destroy. + */ +static void +mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq) +{ + mlx5_devx_sq_destroy(&sq->sq_obj); + mlx5_aso_cq_destroy(&sq->cq); + memset(sq, 0, sizeof(*sq)); +} + +/** + * Initialize Send Queue used for ASO access. + * + * @param[in] sq + * ASO SQ to initialize. + */ +static void +mlx5_aso_age_init_sq(struct mlx5_aso_sq *sq) +{ + volatile struct mlx5_aso_wqe *restrict wqe; + int i; + int size = 1 << sq->log_desc_n; + uint64_t addr; + + /* All the next fields state should stay constant. */ + for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) { + wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | + (sizeof(*wqe) >> 4)); + wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.mkey->id); + addr = (uint64_t)((uint64_t *)sq->mr.buf + i * + MLX5_ASO_AGE_ACTIONS_PER_POOL / 64); + wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32)); + wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u); + wqe->aso_cseg.operand_masks = rte_cpu_to_be_32 + (0u | + (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | + (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) | + (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) | + (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); + wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX); + } +} + +/** + * Initialize Send Queue used for ASO flow meter access. + * + * @param[in] sq + * ASO SQ to initialize. + */ +static void +mlx5_aso_mtr_init_sq(struct mlx5_aso_sq *sq) +{ + volatile struct mlx5_aso_wqe *restrict wqe; + int i; + int size = 1 << sq->log_desc_n; + uint32_t idx; + + /* All the next fields state should stay constant. */ + for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) { + wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | + (sizeof(*wqe) >> 4)); + wqe->aso_cseg.operand_masks = RTE_BE32(0u | + (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | + (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) | + (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) | + (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); + wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << + MLX5_COMP_MODE_OFFSET); + for (idx = 0; idx < MLX5_ASO_METERS_PER_WQE; + idx++) + wqe->aso_dseg.mtrs[idx].v_bo_sc_bbog_mm = + RTE_BE32((1 << ASO_DSEG_VALID_OFFSET) | + (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET)); + } +} + +/** + * Create Send Queue used for ASO access. + * + * @param[in] ctx + * Context returned from mlx5 open_device() glue function. + * @param[in/out] sq + * Pointer to SQ to create. + * @param[in] socket + * Socket to use for allocation. + * @param[in] uar + * User Access Region object. + * @param[in] pdn + * Protection Domain number to use. + * @param[in] log_desc_n + * Log of number of descriptors in queue. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, + void *uar, uint32_t pdn, uint16_t log_desc_n, + uint32_t ts_format) +{ + struct mlx5_devx_create_sq_attr attr = { + .user_index = 0xFFFF, + .wq_attr = (struct mlx5_devx_wq_attr){ + .pd = pdn, + .uar_page = mlx5_os_get_devx_uar_page_id(uar), + }, + .ts_format = mlx5_ts_format_conv(ts_format), + }; + struct mlx5_devx_modify_sq_attr modify_attr = { + .state = MLX5_SQC_STATE_RDY, + }; + uint16_t log_wqbb_n; + int ret; + + if (mlx5_aso_cq_create(ctx, &sq->cq, log_desc_n, socket, + mlx5_os_get_devx_uar_page_id(uar))) + goto error; + sq->log_desc_n = log_desc_n; + attr.cqn = sq->cq.cq_obj.cq->id; + /* for mlx5_aso_wqe that is twice the size of mlx5_wqe */ + log_wqbb_n = log_desc_n + 1; + ret = mlx5_devx_sq_create(ctx, &sq->sq_obj, log_wqbb_n, &attr, socket); + if (ret) { + DRV_LOG(ERR, "Can't create SQ object."); + rte_errno = ENOMEM; + goto error; + } + ret = mlx5_devx_cmd_modify_sq(sq->sq_obj.sq, &modify_attr); + if (ret) { + DRV_LOG(ERR, "Can't change SQ state to ready."); + rte_errno = ENOMEM; + goto error; + } + sq->pi = 0; + sq->head = 0; + sq->tail = 0; + sq->sqn = sq->sq_obj.sq->id; + sq->uar_addr = mlx5_os_get_devx_uar_reg_addr(uar); + return 0; +error: + mlx5_aso_destroy_sq(sq); + return -1; +} + +/** + * API to create and initialize Send Queue used for ASO access. + * + * @param[in] sh + * Pointer to shared device context. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod) +{ + uint32_t sq_desc_n = 1 << MLX5_ASO_QUEUE_LOG_DESC; + + switch (aso_opc_mod) { + case ASO_OPC_MOD_FLOW_HIT: + if (mlx5_aso_devx_reg_mr(sh->ctx, + (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * + sq_desc_n, &sh->aso_age_mng->aso_sq.mr, 0, sh->pdn)) + return -1; + if (mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0, + sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC, + sh->sq_ts_format)) { + mlx5_aso_devx_dereg_mr(&sh->aso_age_mng->aso_sq.mr); + return -1; + } + mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq); + break; + case ASO_OPC_MOD_POLICER: + if (mlx5_aso_sq_create(sh->ctx, &sh->mtrmng->sq, 0, + sh->tx_uar, sh->pdn, MLX5_ASO_QUEUE_LOG_DESC, + sh->sq_ts_format)) + return -1; + mlx5_aso_mtr_init_sq(&sh->mtrmng->sq); + break; + default: + DRV_LOG(ERR, "Unknown ASO operation mode"); + return -1; + } + return 0; +} + +/** + * API to destroy Send Queue used for ASO access. + * + * @param[in] sh + * Pointer to shared device context. + */ +void +mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod) +{ + struct mlx5_aso_sq *sq; + + switch (aso_opc_mod) { + case ASO_OPC_MOD_FLOW_HIT: + mlx5_aso_devx_dereg_mr(&sh->aso_age_mng->aso_sq.mr); + sq = &sh->aso_age_mng->aso_sq; + break; + case ASO_OPC_MOD_POLICER: + sq = &sh->mtrmng->sq; + break; + default: + DRV_LOG(ERR, "Unknown ASO operation mode"); + return; + } + mlx5_aso_destroy_sq(sq); +} + +/** + * Write a burst of WQEs to ASO SQ. + * + * @param[in] mng + * ASO management data, contains the SQ. + * @param[in] n + * Index of the last valid pool. + * + * @return + * Number of WQEs in burst. + */ +static uint16_t +mlx5_aso_sq_enqueue_burst(struct mlx5_aso_age_mng *mng, uint16_t n) +{ + volatile struct mlx5_aso_wqe *wqe; + struct mlx5_aso_sq *sq = &mng->aso_sq; + struct mlx5_aso_age_pool *pool; + uint16_t size = 1 << sq->log_desc_n; + uint16_t mask = size - 1; + uint16_t max; + uint16_t start_head = sq->head; + + max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), n - sq->next); + if (unlikely(!max)) + return 0; + sq->elts[start_head & mask].burst_size = max; + do { + wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; + rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); + /* Fill next WQE. */ + rte_spinlock_lock(&mng->resize_sl); + pool = mng->pools[sq->next]; + rte_spinlock_unlock(&mng->resize_sl); + sq->elts[sq->head & mask].pool = pool; + wqe->general_cseg.misc = + rte_cpu_to_be_32(((struct mlx5_devx_obj *) + (pool->flow_hit_aso_obj))->id); + wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << + MLX5_COMP_MODE_OFFSET); + wqe->general_cseg.opcode = rte_cpu_to_be_32 + (MLX5_OPCODE_ACCESS_ASO | + (ASO_OPC_MOD_FLOW_HIT << + WQE_CSEG_OPC_MOD_OFFSET) | + (sq->pi << + WQE_CSEG_WQE_INDEX_OFFSET)); + sq->pi += 2; /* Each WQE contains 2 WQEBB's. */ + sq->head++; + sq->next++; + max--; + } while (max); + wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << + MLX5_COMP_MODE_OFFSET); + rte_io_wmb(); + sq->sq_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi); + rte_wmb(); + *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH.*/ + rte_wmb(); + return sq->elts[start_head & mask].burst_size; +} + +/** + * Debug utility function. Dump contents of error CQE and WQE. + * + * @param[in] cqe + * Error CQE to dump. + * @param[in] wqe + * Error WQE to dump. + */ +static void +mlx5_aso_dump_err_objs(volatile uint32_t *cqe, volatile uint32_t *wqe) +{ + int i; + + DRV_LOG(ERR, "Error cqe:"); + for (i = 0; i < 16; i += 4) + DRV_LOG(ERR, "%08X %08X %08X %08X", cqe[i], cqe[i + 1], + cqe[i + 2], cqe[i + 3]); + DRV_LOG(ERR, "\nError wqe:"); + for (i = 0; i < (int)sizeof(struct mlx5_aso_wqe) / 4; i += 4) + DRV_LOG(ERR, "%08X %08X %08X %08X", wqe[i], wqe[i + 1], + wqe[i + 2], wqe[i + 3]); +} + +/** + * Handle case of error CQE. + * + * @param[in] sq + * ASO SQ to use. + */ +static void +mlx5_aso_cqe_err_handle(struct mlx5_aso_sq *sq) +{ + struct mlx5_aso_cq *cq = &sq->cq; + uint32_t idx = cq->cq_ci & ((1 << cq->log_desc_n) - 1); + volatile struct mlx5_err_cqe *cqe = + (volatile struct mlx5_err_cqe *)&cq->cq_obj.cqes[idx]; + + cq->errors++; + idx = rte_be_to_cpu_16(cqe->wqe_counter) & (1u << sq->log_desc_n); + mlx5_aso_dump_err_objs((volatile uint32_t *)cqe, + (volatile uint32_t *)&sq->sq_obj.aso_wqes[idx]); +} + +/** + * Update ASO objects upon completion. + * + * @param[in] sh + * Shared device context. + * @param[in] n + * Number of completed ASO objects. + */ +static void +mlx5_aso_age_action_update(struct mlx5_dev_ctx_shared *sh, uint16_t n) +{ + struct mlx5_aso_age_mng *mng = sh->aso_age_mng; + struct mlx5_aso_sq *sq = &mng->aso_sq; + struct mlx5_age_info *age_info; + const uint16_t size = 1 << sq->log_desc_n; + const uint16_t mask = size - 1; + const uint64_t curr = MLX5_CURR_TIME_SEC; + uint16_t expected = AGE_CANDIDATE; + uint16_t i; + + for (i = 0; i < n; ++i) { + uint16_t idx = (sq->tail + i) & mask; + struct mlx5_aso_age_pool *pool = sq->elts[idx].pool; + uint64_t diff = curr - pool->time_of_last_age_check; + uint64_t *addr = sq->mr.buf; + int j; + + addr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64; + pool->time_of_last_age_check = curr; + for (j = 0; j < MLX5_ASO_AGE_ACTIONS_PER_POOL; j++) { + struct mlx5_aso_age_action *act = &pool->actions[j]; + struct mlx5_age_param *ap = &act->age_params; + uint8_t byte; + uint8_t offset; + uint8_t *u8addr; + uint8_t hit; + + if (__atomic_load_n(&ap->state, __ATOMIC_RELAXED) != + AGE_CANDIDATE) + continue; + byte = 63 - (j / 8); + offset = j % 8; + u8addr = (uint8_t *)addr; + hit = (u8addr[byte] >> offset) & 0x1; + if (hit) { + __atomic_store_n(&ap->sec_since_last_hit, 0, + __ATOMIC_RELAXED); + } else { + struct mlx5_priv *priv; + + __atomic_fetch_add(&ap->sec_since_last_hit, + diff, __ATOMIC_RELAXED); + /* If timeout passed add to aged-out list. */ + if (ap->sec_since_last_hit <= ap->timeout) + continue; + priv = + rte_eth_devices[ap->port_id].data->dev_private; + age_info = GET_PORT_AGE_INFO(priv); + rte_spinlock_lock(&age_info->aged_sl); + if (__atomic_compare_exchange_n(&ap->state, + &expected, + AGE_TMOUT, + false, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED)) { + LIST_INSERT_HEAD(&age_info->aged_aso, + act, next); + MLX5_AGE_SET(age_info, + MLX5_AGE_EVENT_NEW); + } + rte_spinlock_unlock(&age_info->aged_sl); + } + } + } + mlx5_age_event_prepare(sh); +} + +/** + * Handle completions from WQEs sent to ASO SQ. + * + * @param[in] sh + * Shared device context. + * + * @return + * Number of CQEs handled. + */ +static uint16_t +mlx5_aso_completion_handle(struct mlx5_dev_ctx_shared *sh) +{ + struct mlx5_aso_age_mng *mng = sh->aso_age_mng; + struct mlx5_aso_sq *sq = &mng->aso_sq; + struct mlx5_aso_cq *cq = &sq->cq; + volatile struct mlx5_cqe *restrict cqe; + const unsigned int cq_size = 1 << cq->log_desc_n; + const unsigned int mask = cq_size - 1; + uint32_t idx; + uint32_t next_idx = cq->cq_ci & mask; + const uint16_t max = (uint16_t)(sq->head - sq->tail); + uint16_t i = 0; + int ret; + if (unlikely(!max)) + return 0; + do { + idx = next_idx; + next_idx = (cq->cq_ci + 1) & mask; + rte_prefetch0(&cq->cq_obj.cqes[next_idx]); + cqe = &cq->cq_obj.cqes[idx]; + ret = check_cqe(cqe, cq_size, cq->cq_ci); + /* + * Be sure owner read is done before any other cookie field or + * opaque field. + */ + rte_io_rmb(); + if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { + if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) + break; + mlx5_aso_cqe_err_handle(sq); + } else { + i += sq->elts[(sq->tail + i) & mask].burst_size; + } + cq->cq_ci++; + } while (1); + if (likely(i)) { + mlx5_aso_age_action_update(sh, i); + sq->tail += i; + rte_io_wmb(); + cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); + } + return i; +} + +/** + * Periodically read CQEs and send WQEs to ASO SQ. + * + * @param[in] arg + * Shared device context containing the ASO SQ. + */ +static void +mlx5_flow_aso_alarm(void *arg) +{ + struct mlx5_dev_ctx_shared *sh = arg; + struct mlx5_aso_sq *sq = &sh->aso_age_mng->aso_sq; + uint32_t us = 100u; + uint16_t n; + + rte_spinlock_lock(&sh->aso_age_mng->resize_sl); + n = sh->aso_age_mng->next; + rte_spinlock_unlock(&sh->aso_age_mng->resize_sl); + mlx5_aso_completion_handle(sh); + if (sq->next == n) { + /* End of loop: wait 1 second. */ + us = US_PER_S; + sq->next = 0; + } + mlx5_aso_sq_enqueue_burst(sh->aso_age_mng, n); + if (rte_eal_alarm_set(us, mlx5_flow_aso_alarm, sh)) + DRV_LOG(ERR, "Cannot reinitialize aso alarm."); +} + +/** + * API to start ASO access using ASO SQ. + * + * @param[in] sh + * Pointer to shared device context. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh) +{ + if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) { + DRV_LOG(ERR, "Cannot reinitialize ASO age alarm."); + return -rte_errno; + } + return 0; +} + +/** + * API to stop ASO access using ASO SQ. + * + * @param[in] sh + * Pointer to shared device context. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh) +{ + int retries = 1024; + + if (!sh->aso_age_mng->aso_sq.sq_obj.sq) + return -EINVAL; + rte_errno = 0; + while (--retries) { + rte_eal_alarm_cancel(mlx5_flow_aso_alarm, sh); + if (rte_errno != EINPROGRESS) + break; + rte_pause(); + } + return -rte_errno; +} diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 4b2a272..a621417 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -5962,6 +5962,11 @@ struct mlx5_hlist_entry * rte_errno = ENOMEM; return -ENOMEM; } + if (!mtrmng->n) + if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) { + mlx5_free(pools); + return -ENOMEM; + } if (old_pools) memcpy(pools, old_pools, mtrmng->n * sizeof(struct mlx5_aso_mtr_pool *)); @@ -10834,7 +10839,7 @@ struct mlx5_cache_entry * mlx5_free(old_pools); } else { /* First ASO flow hit allocation - starting ASO data-path. */ - int ret = mlx5_aso_queue_start(priv->sh); + int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh); if (ret) { mlx5_free(pools); diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index 956a6c3..ef4ca30 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -811,7 +811,6 @@ struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, @@ -836,7 +835,7 @@ RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter object is being used."); if (priv->sh->meter_aso_en) { - if (mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id)) + if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id)) return -rte_mtr_error_set(error, EBUSY, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Fail to delete ASO Meter in index table."); @@ -1302,7 +1301,7 @@ struct mlx5_flow_meter_info * rte_spinlock_unlock(&mtrmng->mtrsl); return NULL; } - if (mlx5_l3t_get_entry(mtrmng->mtr_idx_tbl, meter_id, &data) || + if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || !data.dword) { rte_spinlock_unlock(&mtrmng->mtrsl); return NULL; @@ -1310,7 +1309,7 @@ struct mlx5_flow_meter_info * if (mtr_idx) *mtr_idx = data.dword; aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); - mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id); + mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); if (meter_id == aso_mtr->fm.meter_id) { rte_spinlock_unlock(&mtrmng->mtrsl); return &aso_mtr->fm;