From patchwork Mon Jan 22 12:33:38 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Olivier Matz X-Patchwork-Id: 34229 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id E2C30A48B; Mon, 22 Jan 2018 13:34:15 +0100 (CET) Received: from proxy.6wind.com (host.76.145.23.62.rev.coltfrance.com [62.23.145.76]) by dpdk.org (Postfix) with ESMTP id 8AE2B324D; Mon, 22 Jan 2018 13:34:14 +0100 (CET) Received: from glumotte.dev.6wind.com. (unknown [10.16.0.195]) by proxy.6wind.com (Postfix) with ESMTP id 8CA6C125427; Mon, 22 Jan 2018 13:32:29 +0100 (CET) From: Olivier Matz To: dev@dpdk.org, Adrien Mazarguil , Nelio Laranjeiro , Yongseok Koh Cc: stable@dpdk.org Date: Mon, 22 Jan 2018 13:33:38 +0100 Message-Id: <20180122123338.19806-2-olivier.matz@6wind.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180122123338.19806-1-olivier.matz@6wind.com> References: <20180119162517.4451-1-olivier.matz@6wind.com> <20180122123338.19806-1-olivier.matz@6wind.com> Subject: [dpdk-dev] [PATCH v3 2/2] net/mlx5: fix allocation when no memory on device NUMA node X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" When no memory is available on the same numa node than the device, the initialization of the device fails. However, the use case where the cores and memory are on a different socket than the device is valid, even if not optimal. To fix this issue, this commit introduces an infrastructure to select the socket on which to allocate the verbs objects based on the ethdev configuration and the object type, rather than the PCI numa node. Fixes: 1e3a39f72d5d ("net/mlx5: allocate verbs object into shared memory") Cc: stable@dpdk.org Signed-off-by: Olivier Matz Signed-off-by: Nelio Laranjeiro --- v2->v3: - fix commit log - VERSB -> VERBS v1->v2: - update from Nelop to select socket on which to allocate based on config and object type drivers/net/mlx5/mlx5.c | 14 ++++++++++++-- drivers/net/mlx5/mlx5.h | 20 ++++++++++++++++++++ drivers/net/mlx5/mlx5_rxq.c | 4 ++++ drivers/net/mlx5/mlx5_txq.c | 4 ++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 9d1de366b..bc4b6bad0 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -139,10 +139,20 @@ mlx5_alloc_verbs_buf(size_t size, void *data) struct priv *priv = data; void *ret; size_t alignment = sysconf(_SC_PAGESIZE); + unsigned int socket = SOCKET_ID_ANY; + if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) { + const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; + + socket = ctrl->socket; + } else if (priv->verbs_alloc_ctx.type == + MLX5_VERBS_ALLOC_TYPE_RX_QUEUE) { + const struct mlx5_rxq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; + + socket = ctrl->socket; + } assert(data != NULL); - ret = rte_malloc_socket(__func__, size, alignment, - priv->dev->device->numa_node); + ret = rte_malloc_socket(__func__, size, alignment, socket); DEBUG("Extern alloc size: %lu, align: %lu: %p", size, alignment, ret); return ret; } diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index eb0894fa5..a7ec607c3 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -123,6 +123,24 @@ struct mlx5_dev_config { int inline_max_packet_sz; /* Max packet size for inlining. */ }; +/** + * Type of objet being allocated. + */ +enum mlx5_verbs_alloc_type { + MLX5_VERBS_ALLOC_TYPE_NONE, + MLX5_VERBS_ALLOC_TYPE_TX_QUEUE, + MLX5_VERBS_ALLOC_TYPE_RX_QUEUE, +}; + +/** + * Verbs allocator needs a context to know in the callback which kind of + * resources it is allocating. + */ +struct mlx5_verbs_alloc_ctx { + enum mlx5_verbs_alloc_type type; /* Kind of object being allocated. */ + const void *obj; /* Pointer to the DPDK object. */ +}; + struct priv { struct rte_eth_dev *dev; /* Ethernet device of master process. */ struct ibv_context *ctx; /* Verbs context. */ @@ -164,6 +182,8 @@ struct priv { int primary_socket; /* Unix socket for primary process. */ struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */ struct mlx5_dev_config config; /* Device configuration. */ + struct mlx5_verbs_alloc_ctx verbs_alloc_ctx; + /* Context for Verbs allocator. */ }; /** diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 8e4fbbf2a..0274ccf31 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -655,6 +655,8 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) assert(rxq_data); assert(!rxq_ctrl->ibv); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_RX_QUEUE; + priv->verbs_alloc_ctx.obj = rxq_ctrl; tmpl = rte_calloc_socket(__func__, 1, sizeof(*tmpl), 0, rxq_ctrl->socket); if (!tmpl) { @@ -818,6 +820,7 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); LIST_INSERT_HEAD(&priv->rxqsibv, tmpl, next); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; return tmpl; error: if (tmpl->wq) @@ -828,6 +831,7 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) claim_zero(ibv_destroy_comp_channel(tmpl->channel)); if (tmpl->mr) priv_mr_release(priv, tmpl->mr); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; return NULL; } diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 49018303e..18321c3e3 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -396,6 +396,8 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) int ret = 0; assert(txq_data); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_TX_QUEUE; + priv->verbs_alloc_ctx.obj = txq_ctrl; if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set"); goto error; @@ -526,12 +528,14 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; return txq_ibv; error: if (tmpl.cq) claim_zero(ibv_destroy_cq(tmpl.cq)); if (tmpl.qp) claim_zero(ibv_destroy_qp(tmpl.qp)); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; return NULL; }