From patchwork Tue Oct 6 11:48:55 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 79767 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8171CA04BB; Tue, 6 Oct 2020 13:54:02 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id B40D51B9E6; Tue, 6 Oct 2020 13:49:44 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 089181B870 for ; Tue, 6 Oct 2020 13:49:39 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from suanmingm@nvidia.com) with SMTP; 6 Oct 2020 14:49:35 +0300 Received: from nvidia.com (mtbc-r640-04.mtbc.labs.mlnx [10.75.70.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 096BnC0W028553; Tue, 6 Oct 2020 14:49:34 +0300 From: Suanming Mou To: viacheslavo@nvidia.com, matan@nvidia.com Cc: rasland@nvidia.com, dev@dpdk.org, Xueming Li Date: Tue, 6 Oct 2020 19:48:55 +0800 Message-Id: <1601984948-313027-13-git-send-email-suanmingm@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1601984948-313027-1-git-send-email-suanmingm@nvidia.com> References: <1601984948-313027-1-git-send-email-suanmingm@nvidia.com> Subject: [dpdk-dev] [PATCH 12/25] net/mlx5: make metadata copy flow list thread safe X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Xueming Li To support multi-thread flow insertion, this patch updates metadata copy flow list to use thread safe hash list. Signed-off-by: Xueming Li --- drivers/net/mlx5/linux/mlx5_os.c | 6 +- drivers/net/mlx5/mlx5_flow.c | 276 +++++++++++++++++++-------------------- drivers/net/mlx5/mlx5_flow.h | 6 +- 3 files changed, 139 insertions(+), 149 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 744b1dd..50d3d99 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -1354,13 +1354,13 @@ mlx5_flow_ext_mreg_supported(eth_dev) && priv->sh->dv_regc0_mask) { priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME, - MLX5_FLOW_MREG_HTABLE_SZ, - 0, false, - NULL, NULL, NULL); + MLX5_FLOW_MREG_HTABLE_SZ, 0, false, + flow_dv_mreg_create_cb, NULL, flow_dv_mreg_remove_cb); if (!priv->mreg_cp_tbl) { err = ENOMEM; goto error; } + priv->mreg_cp_tbl->ctx = eth_dev; } return eth_dev; error: diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 3a3b783..3808e2b 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -2950,36 +2950,17 @@ struct mlx5_flow_tunnel_info { flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, uint32_t flow_idx); -/** - * Add a flow of copying flow metadata registers in RX_CP_TBL. - * - * As mark_id is unique, if there's already a registered flow for the mark_id, - * return by increasing the reference counter of the resource. Otherwise, create - * the resource (mcp_res) and flow. - * - * Flow looks like, - * - If ingress port is ANY and reg_c[1] is mark_id, - * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. - * - * For default flow (zero mark_id), flow is like, - * - If ingress port is ANY, - * reg_b := reg_c[0] and jump to RX_ACT_TBL. - * - * @param dev - * Pointer to Ethernet device. - * @param mark_id - * ID of MARK action, zero means default flow for META. - * @param[out] error - * Perform verbose error reporting if not NULL. - * - * @return - * Associated resource on success, NULL otherwise and rte_errno is set. - */ -static struct mlx5_flow_mreg_copy_resource * -flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, - struct rte_flow_error *error) +struct mlx5_hlist_entry * +flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, + void *cb_ctx) { + struct rte_eth_dev *dev = list->ctx; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_mreg_copy_resource *mcp_res; + uint32_t idx = 0; + int ret; + uint32_t mark_id = key; struct rte_flow_attr attr = { .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, .ingress = 1, @@ -3003,30 +2984,22 @@ struct mlx5_flow_tunnel_info { struct rte_flow_action actions[] = { [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, }; - struct mlx5_flow_mreg_copy_resource *mcp_res; - uint32_t idx = 0; - int ret; /* Fill the register fileds in the flow. */ - ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); + ret = mlx5_flow_get_reg_id(ctx->dev, MLX5_FLOW_MARK, 0, ctx->error); if (ret < 0) return NULL; tag_spec.id = ret; - ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); + ret = mlx5_flow_get_reg_id(ctx->dev, MLX5_METADATA_RX, 0, ctx->error); if (ret < 0) return NULL; cp_mreg.src = ret; - /* Check if already registered. */ - MLX5_ASSERT(priv->mreg_cp_tbl); - mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, NULL); - if (mcp_res) { - /* For non-default rule. */ - if (mark_id != MLX5_DEFAULT_COPY_ID) - mcp_res->refcnt++; - MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || - mcp_res->refcnt == 1); - return mcp_res; + mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); + if (!mcp_res) { + rte_errno = ENOMEM; + return NULL; } + mcp_res->idx = idx; /* Provide the full width of FLAG specific value. */ if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) tag_spec.data = MLX5_FLOW_MARK_DEFAULT; @@ -3076,39 +3049,68 @@ struct mlx5_flow_tunnel_info { .type = RTE_FLOW_ACTION_TYPE_END, }; } - /* Build a new entry. */ - mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); - if (!mcp_res) { - rte_errno = ENOMEM; - return NULL; - } - mcp_res->idx = idx; /* * The copy Flows are not included in any list. There * ones are referenced from other Flows and can not - * be applied, removed, deleted in ardbitrary order + * be applied, removed, deleted in arbitrary order * by list traversing. */ - mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, - actions, false, error); - if (!mcp_res->rix_flow) - goto error; - mcp_res->refcnt++; - mcp_res->hlist_ent.key = mark_id; - ret = !mlx5_hlist_insert(priv->mreg_cp_tbl, &mcp_res->hlist_ent); - MLX5_ASSERT(!ret); - if (ret) - goto error; - return mcp_res; -error: - if (mcp_res->rix_flow) - flow_list_destroy(dev, NULL, mcp_res->rix_flow); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); - return NULL; + mcp_res->rix_flow = flow_list_create(ctx->dev, NULL, &attr, items, + actions, false, ctx->error); + if (!mcp_res->rix_flow) { + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); + return NULL; + } + return &mcp_res->hlist_ent; } /** - * Release flow in RX_CP_TBL. + * Add a flow of copying flow metadata registers in RX_CP_TBL. + * + * As mark_id is unique, if there's already a registered flow for the mark_id, + * return by increasing the reference counter of the resource. Otherwise, create + * the resource (mcp_res) and flow. + * + * Flow looks like, + * - If ingress port is ANY and reg_c[1] is mark_id, + * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. + * + * For default flow (zero mark_id), flow is like, + * - If ingress port is ANY, + * reg_b := reg_c[0] and jump to RX_ACT_TBL. + * + * @param dev + * Pointer to Ethernet device. + * @param mark_id + * ID of MARK action, zero means default flow for META. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * Associated resource on success, NULL otherwise and rte_errno is set. + */ +static struct mlx5_flow_mreg_copy_resource * +flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_hlist_entry *entry; + struct mlx5_flow_cb_ctx ctx = { + .dev = dev, + .error = error, + }; + + /* Check if already registered. */ + MLX5_ASSERT(priv->mreg_cp_tbl); + entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); + if (!entry) + return NULL; + return container_of(entry, struct mlx5_flow_mreg_copy_resource, + hlist_ent); +} + +/** + * Stop flow in RX_CP_TBL. * * @param dev * Pointer to Ethernet device. @@ -3116,117 +3118,102 @@ struct mlx5_flow_tunnel_info { * Parent flow for wich copying is provided. */ static void -flow_mreg_del_copy_action(struct rte_eth_dev *dev, - struct rte_flow *flow) +flow_mreg_stop_copy_action(struct rte_eth_dev *dev, + struct rte_flow *flow) { struct mlx5_flow_mreg_copy_resource *mcp_res; struct mlx5_priv *priv = dev->data->dev_private; + struct rte_flow *mcp_flow; - if (!flow->rix_mreg_copy) + if (!flow->rix_mreg_copy || !flow->copy_applied) return; mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], flow->rix_mreg_copy); - if (!mcp_res || !priv->mreg_cp_tbl) + if (!mcp_res) return; - if (flow->copy_applied) { - MLX5_ASSERT(mcp_res->appcnt); - flow->copy_applied = 0; - --mcp_res->appcnt; - if (!mcp_res->appcnt) { - struct rte_flow *mcp_flow = mlx5_ipool_get - (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], - mcp_res->rix_flow); - - if (mcp_flow) - flow_drv_remove(dev, mcp_flow); - } - } - /* - * We do not check availability of metadata registers here, - * because copy resources are not allocated in this case. - */ - if (--mcp_res->refcnt) + if (__atomic_sub_fetch(&mcp_res->appcnt, 0, __ATOMIC_ACQ_REL)) return; + flow->copy_applied = 0; + mcp_flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], + mcp_res->rix_flow); + if (mcp_flow) + flow_drv_remove(dev, mcp_flow); +} + +void +flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) +{ + struct mlx5_flow_mreg_copy_resource *mcp_res = + container_of(entry, typeof(*mcp_res), hlist_ent); + struct rte_eth_dev *dev = list->ctx; + struct mlx5_priv *priv = dev->data->dev_private; + MLX5_ASSERT(mcp_res->rix_flow); flow_list_destroy(dev, NULL, mcp_res->rix_flow); - mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); - flow->rix_mreg_copy = 0; } /** - * Start flow in RX_CP_TBL. + * Release flow in RX_CP_TBL. * * @param dev * Pointer to Ethernet device. * @flow * Parent flow for wich copying is provided. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. */ -static int -flow_mreg_start_copy_action(struct rte_eth_dev *dev, - struct rte_flow *flow) +static void +flow_mreg_del_copy_action(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_mreg_copy_resource *mcp_res; struct mlx5_priv *priv = dev->data->dev_private; - int ret; - if (!flow->rix_mreg_copy || flow->copy_applied) - return 0; + if (!flow->rix_mreg_copy) + return; mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], flow->rix_mreg_copy); - if (!mcp_res) - return 0; - if (!mcp_res->appcnt) { - struct rte_flow *mcp_flow = mlx5_ipool_get - (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], - mcp_res->rix_flow); - - if (mcp_flow) { - ret = flow_drv_apply(dev, mcp_flow, NULL); - if (ret) - return ret; - } - } - ++mcp_res->appcnt; - flow->copy_applied = 1; - return 0; + if (!mcp_res || !priv->mreg_cp_tbl) + return; + flow_mreg_stop_copy_action(dev, flow); + mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); } /** - * Stop flow in RX_CP_TBL. + * Start flow in RX_CP_TBL. * * @param dev * Pointer to Ethernet device. * @flow * Parent flow for wich copying is provided. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -static void -flow_mreg_stop_copy_action(struct rte_eth_dev *dev, - struct rte_flow *flow) +static int +flow_mreg_start_copy_action(struct rte_eth_dev *dev, + struct rte_flow *flow) { struct mlx5_flow_mreg_copy_resource *mcp_res; struct mlx5_priv *priv = dev->data->dev_private; + int ret; - if (!flow->rix_mreg_copy || !flow->copy_applied) - return; + if (!flow->rix_mreg_copy || flow->copy_applied) + return 0; mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], flow->rix_mreg_copy); if (!mcp_res) - return; - MLX5_ASSERT(mcp_res->appcnt); - --mcp_res->appcnt; - flow->copy_applied = 0; - if (!mcp_res->appcnt) { - struct rte_flow *mcp_flow = mlx5_ipool_get - (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], - mcp_res->rix_flow); - - if (mcp_flow) - flow_drv_remove(dev, mcp_flow); + return 0; + if (__atomic_fetch_add(&mcp_res->appcnt, 1, __ATOMIC_ACQ_REL)) + return 0; + struct rte_flow *mcp_flow = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], + mcp_res->rix_flow); + if (mcp_flow) { + ret = flow_drv_apply(dev, mcp_flow, NULL); + if (ret) + return ret; } + flow->copy_applied = 1; + return 0; } /** @@ -3238,20 +3225,17 @@ struct mlx5_flow_tunnel_info { static void flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) { - struct mlx5_flow_mreg_copy_resource *mcp_res; + struct mlx5_hlist_entry *entry; struct mlx5_priv *priv = dev->data->dev_private; /* Check if default flow is registered. */ if (!priv->mreg_cp_tbl) return; - mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, - MLX5_DEFAULT_COPY_ID, NULL); - if (!mcp_res) + entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, + MLX5_DEFAULT_COPY_ID, NULL); + if (!entry) return; - MLX5_ASSERT(mcp_res->rix_flow); - flow_list_destroy(dev, NULL, mcp_res->rix_flow); - mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); + mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); } /** @@ -3345,7 +3329,8 @@ struct mlx5_flow_tunnel_info { return -rte_errno; flow->rix_mreg_copy = mcp_res->idx; if (dev->data->dev_started) { - mcp_res->appcnt++; + __atomic_add_fetch(&mcp_res->appcnt, 1, + __ATOMIC_RELAXED); flow->copy_applied = 1; } return 0; @@ -3358,7 +3343,8 @@ struct mlx5_flow_tunnel_info { return -rte_errno; flow->rix_mreg_copy = mcp_res->idx; if (dev->data->dev_started) { - mcp_res->appcnt++; + __atomic_add_fetch(&mcp_res->appcnt, 1, + __ATOMIC_RELAXED); flow->copy_applied = 1; } return 0; diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 7effacc..41969c2 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -508,7 +508,6 @@ struct mlx5_flow_mreg_copy_resource { struct mlx5_hlist_entry hlist_ent; LIST_ENTRY(mlx5_flow_mreg_copy_resource) next; /* List entry for device flows. */ - uint32_t refcnt; /* Reference counter. */ uint32_t appcnt; /* Apply/Remove counter. */ uint32_t idx; uint32_t rix_flow; /* Built flow for copy. */ @@ -1101,4 +1100,9 @@ struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list, void flow_dv_modify_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry); +struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list, + uint64_t key, void *ctx); +void flow_dv_mreg_remove_cb(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry); + #endif /* RTE_PMD_MLX5_FLOW_H_ */