From patchwork Thu Dec 17 11:44:27 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Baum X-Patchwork-Id: 85317 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E26EFA09F6; Thu, 17 Dec 2020 12:48:08 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 63C34CA8F; Thu, 17 Dec 2020 12:45:13 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id C8511CA08 for ; Thu, 17 Dec 2020 12:44:56 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from michaelba@nvidia.com) with SMTP; 17 Dec 2020 13:44:50 +0200 Received: from nvidia.com (pegasus07.mtr.labs.mlnx [10.210.16.112]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BHBio2Y004524; Thu, 17 Dec 2020 13:44:50 +0200 From: Michael Baum To: dev@dpdk.org Cc: Matan Azrad , Raslan Darawsheh , Viacheslav Ovsiienko Date: Thu, 17 Dec 2020 11:44:27 +0000 Message-Id: <1608205475-20067-10-git-send-email-michaelba@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1608205475-20067-1-git-send-email-michaelba@nvidia.com> References: <1608205475-20067-1-git-send-email-michaelba@nvidia.com> Subject: [dpdk-dev] [PATCH 09/17] common/mlx5: enhance page size configuration X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" The PRM calculates page size in 4K, so need to reduce the log_wq_pg_sz attribute. Signed-off-by: Michael Baum Acked-by: Matan Azrad --- drivers/common/mlx5/mlx5_devx_cmds.c | 53 ++++++++++++++++-------------------- drivers/net/mlx5/mlx5_devx.c | 13 +++++---- 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index 9c1d188..09e204b 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -268,9 +268,8 @@ struct mlx5_devx_obj * MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF); MLX5_SET(mkc, mkc, translations_octword_size, translation_size); MLX5_SET(mkc, mkc, relaxed_ordering_write, - attr->relaxed_ordering_write); - MLX5_SET(mkc, mkc, relaxed_ordering_read, - attr->relaxed_ordering_read); + attr->relaxed_ordering_write); + MLX5_SET(mkc, mkc, relaxed_ordering_read, attr->relaxed_ordering_read); MLX5_SET64(mkc, mkc, start_addr, attr->addr); MLX5_SET64(mkc, mkc, len, attr->size); mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out, @@ -308,7 +307,7 @@ struct mlx5_devx_obj * if (status) { int syndrome = MLX5_GET(query_flow_counter_out, out, syndrome); - DRV_LOG(ERR, "Bad devX status %x, syndrome = %x", status, + DRV_LOG(ERR, "Bad DevX status %x, syndrome = %x", status, syndrome); } return status; @@ -374,8 +373,7 @@ struct mlx5_devx_obj * syndrome = MLX5_GET(query_nic_vport_context_out, out, syndrome); if (status) { DRV_LOG(DEBUG, "Failed to query NIC vport context, " - "status %x, syndrome = %x", - status, syndrome); + "status %x, syndrome = %x", status, syndrome); return -1; } vctx = MLX5_ADDR_OF(query_nic_vport_context_out, out, @@ -662,8 +660,7 @@ struct mlx5_devx_obj * syndrome = MLX5_GET(query_hca_cap_out, out, syndrome); if (status) { DRV_LOG(DEBUG, "Failed to query devx HCA capabilities, " - "status %x, syndrome = %x", - status, syndrome); + "status %x, syndrome = %x", status, syndrome); return -1; } hcattr = MLX5_ADDR_OF(query_hca_cap_out, out, capability); @@ -683,11 +680,11 @@ struct mlx5_devx_obj * (cmd_hca_cap, hcattr, log_min_hairpin_wq_data_sz); attr->vhca_id = MLX5_GET(cmd_hca_cap, hcattr, vhca_id); attr->relaxed_ordering_write = MLX5_GET(cmd_hca_cap, hcattr, - relaxed_ordering_write); + relaxed_ordering_write); attr->relaxed_ordering_read = MLX5_GET(cmd_hca_cap, hcattr, - relaxed_ordering_read); + relaxed_ordering_read); attr->access_register_user = MLX5_GET(cmd_hca_cap, hcattr, - access_register_user); + access_register_user); attr->eth_net_offloads = MLX5_GET(cmd_hca_cap, hcattr, eth_net_offloads); attr->eth_virt = MLX5_GET(cmd_hca_cap, hcattr, eth_virt); @@ -730,8 +727,7 @@ struct mlx5_devx_obj * goto error; if (status) { DRV_LOG(DEBUG, "Failed to query devx QOS capabilities," - " status %x, syndrome = %x", - status, syndrome); + " status %x, syndrome = %x", status, syndrome); return -1; } hcattr = MLX5_ADDR_OF(query_hca_cap_out, out, capability); @@ -761,17 +757,14 @@ struct mlx5_devx_obj * MLX5_GET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE | MLX5_HCA_CAP_OPMOD_GET_CUR); - rc = mlx5_glue->devx_general_cmd(ctx, - in, sizeof(in), - out, sizeof(out)); + rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out)); if (rc) goto error; status = MLX5_GET(query_hca_cap_out, out, status); syndrome = MLX5_GET(query_hca_cap_out, out, syndrome); if (status) { DRV_LOG(DEBUG, "Failed to query devx HCA capabilities, " - "status %x, syndrome = %x", - status, syndrome); + "status %x, syndrome = %x", status, syndrome); attr->log_max_ft_sampler_num = 0; return -1; } @@ -788,9 +781,7 @@ struct mlx5_devx_obj * MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS | MLX5_HCA_CAP_OPMOD_GET_CUR); - rc = mlx5_glue->devx_general_cmd(ctx, - in, sizeof(in), - out, sizeof(out)); + rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out)); if (rc) { attr->eth_net_offloads = 0; goto error; @@ -799,8 +790,7 @@ struct mlx5_devx_obj * syndrome = MLX5_GET(query_hca_cap_out, out, syndrome); if (status) { DRV_LOG(DEBUG, "Failed to query devx HCA capabilities, " - "status %x, syndrome = %x", - status, syndrome); + "status %x, syndrome = %x", status, syndrome); attr->eth_net_offloads = 0; return -1; } @@ -916,7 +906,9 @@ struct mlx5_devx_obj * MLX5_SET(wq, wq_ctx, hw_counter, wq_attr->hw_counter); MLX5_SET(wq, wq_ctx, sw_counter, wq_attr->sw_counter); MLX5_SET(wq, wq_ctx, log_wq_stride, wq_attr->log_wq_stride); - MLX5_SET(wq, wq_ctx, log_wq_pg_sz, wq_attr->log_wq_pg_sz); + if (wq_attr->log_wq_pg_sz > MLX5_ADAPTER_PAGE_SHIFT) + MLX5_SET(wq, wq_ctx, log_wq_pg_sz, + wq_attr->log_wq_pg_sz - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET(wq, wq_ctx, log_wq_sz, wq_attr->log_wq_sz); MLX5_SET(wq, wq_ctx, dbr_umem_valid, wq_attr->dbr_umem_valid); MLX5_SET(wq, wq_ctx, wq_umem_valid, wq_attr->wq_umem_valid); @@ -1562,13 +1554,13 @@ struct mlx5_devx_obj * MLX5_SET(cqc, cqctx, cc, attr->use_first_only); MLX5_SET(cqc, cqctx, oi, attr->overrun_ignore); MLX5_SET(cqc, cqctx, log_cq_size, attr->log_cq_size); - MLX5_SET(cqc, cqctx, log_page_size, attr->log_page_size - - MLX5_ADAPTER_PAGE_SHIFT); + if (attr->log_page_size > MLX5_ADAPTER_PAGE_SHIFT) + MLX5_SET(cqc, cqctx, log_page_size, + attr->log_page_size - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET(cqc, cqctx, c_eqn, attr->eqn); MLX5_SET(cqc, cqctx, uar_page, attr->uar_page_id); MLX5_SET(cqc, cqctx, cqe_comp_en, !!attr->cqe_comp_en); - MLX5_SET(cqc, cqctx, mini_cqe_res_format, - attr->mini_cqe_res_format); + MLX5_SET(cqc, cqctx, mini_cqe_res_format, attr->mini_cqe_res_format); MLX5_SET(cqc, cqctx, mini_cqe_res_format_ext, attr->mini_cqe_res_format_ext); MLX5_SET(cqc, cqctx, cqe_sz, attr->cqe_size); @@ -1798,8 +1790,9 @@ struct mlx5_devx_obj * if (attr->uar_index) { MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); MLX5_SET(qpc, qpc, uar_page, attr->uar_index); - MLX5_SET(qpc, qpc, log_page_size, attr->log_page_size - - MLX5_ADAPTER_PAGE_SHIFT); + if (attr->log_page_size > MLX5_ADAPTER_PAGE_SHIFT) + MLX5_SET(qpc, qpc, log_page_size, + attr->log_page_size - MLX5_ADAPTER_PAGE_SHIFT); if (attr->sq_size) { MLX5_ASSERT(RTE_IS_POWER_OF_2(attr->sq_size)); MLX5_SET(qpc, qpc, cqn_snd, attr->cqn); diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c index 6ad70f2..fe103a7 100644 --- a/drivers/net/mlx5/mlx5_devx.c +++ b/drivers/net/mlx5/mlx5_devx.c @@ -320,7 +320,13 @@ uint32_t log_wqe_size = 0; void *buf = NULL; struct mlx5_devx_obj *rq; + size_t alignment = MLX5_WQE_BUF_ALIGNMENT; + if (alignment == (size_t)-1) { + DRV_LOG(ERR, "Failed to get mem page size"); + rte_errno = ENOMEM; + return NULL; + } /* Fill RQ attributes. */ rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; rq_attr.flush_in_error_en = 1; @@ -347,15 +353,10 @@ log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; rq_attr.wq_attr.log_wq_stride = log_wqe_size; rq_attr.wq_attr.log_wq_sz = rxq_data->elts_n - rxq_data->sges_n; + rq_attr.wq_attr.log_wq_pg_sz = log2above(alignment); /* Calculate and allocate WQ memory space. */ wqe_size = 1 << log_wqe_size; /* round up power of two.*/ wq_size = wqe_n * wqe_size; - size_t alignment = MLX5_WQE_BUF_ALIGNMENT; - if (alignment == (size_t)-1) { - DRV_LOG(ERR, "Failed to get mem page size"); - rte_errno = ENOMEM; - return NULL; - } buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size, alignment, rxq_ctrl->socket); if (!buf)