From patchwork Tue Oct 6 11:49:00 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 79773 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 555FDA04BB; Tue, 6 Oct 2020 13:56:33 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 457F31BB6F; Tue, 6 Oct 2020 13:49:54 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 4689F1BAC5 for ; Tue, 6 Oct 2020 13:49:50 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from suanmingm@nvidia.com) with SMTP; 6 Oct 2020 14:49:44 +0300 Received: from nvidia.com (mtbc-r640-04.mtbc.labs.mlnx [10.75.70.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 096BnC0b028553; Tue, 6 Oct 2020 14:49:42 +0300 From: Suanming Mou To: viacheslavo@nvidia.com, matan@nvidia.com Cc: rasland@nvidia.com, dev@dpdk.org, Xueming Li Date: Tue, 6 Oct 2020 19:49:00 +0800 Message-Id: <1601984948-313027-18-git-send-email-suanmingm@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1601984948-313027-1-git-send-email-suanmingm@nvidia.com> References: <1601984948-313027-1-git-send-email-suanmingm@nvidia.com> Subject: [dpdk-dev] [PATCH 17/25] net/mlx5: make matcher list thread safe X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Xueming Li To support multi-thread flow insertion, this path converts matcher list to use thread safe cache list API. Signed-off-by: Xueming Li --- drivers/net/mlx5/mlx5.h | 3 + drivers/net/mlx5/mlx5_flow.h | 15 ++- drivers/net/mlx5/mlx5_flow_dv.c | 209 +++++++++++++++++++++------------------- 3 files changed, 126 insertions(+), 101 deletions(-) diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 97729a8..ee2211b 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -32,6 +32,9 @@ #include "mlx5_os.h" #include "mlx5_autoconf.h" + +#define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh) + enum mlx5_ipool_index { #ifdef HAVE_IBV_FLOW_DV_SUPPORT MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 6ec0e72..fe9b6a6 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -378,11 +378,9 @@ struct mlx5_flow_dv_match_params { /* Matcher structure. */ struct mlx5_flow_dv_matcher { - LIST_ENTRY(mlx5_flow_dv_matcher) next; - /**< Pointer to the next element. */ + struct mlx5_cache_entry entry; /**< Pointer to the next element. */ struct mlx5_flow_tbl_resource *tbl; /**< Pointer to the table(group) the matcher associated with. */ - rte_atomic32_t refcnt; /**< Reference counter. */ void *matcher_object; /**< Pointer to DV matcher */ uint16_t crc; /**< CRC of key. */ uint16_t priority; /**< Priority of matcher. */ @@ -512,11 +510,12 @@ struct mlx5_flow_tbl_data_entry { /**< hash list entry, 64-bits key inside. */ struct mlx5_flow_tbl_resource tbl; /**< flow table resource. */ - LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers; + struct mlx5_cache_list matchers; /**< matchers' header associated with the flow table. */ struct mlx5_flow_dv_jump_tbl_resource jump; /**< jump resource, at most one for each table created. */ uint32_t idx; /**< index for the indexed mempool. */ + uint8_t direction; /**< table direction, 0: ingress, 1: egress. */ }; /* Verbs specification header. */ @@ -1095,4 +1094,12 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list, uint64_t key, void *cb_ctx); void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry); + +int flow_dv_matcher_match_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry, void *ctx); +struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry, void *ctx); +void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry); + #endif /* RTE_PMD_MLX5_FLOW_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 5092130..3774e46 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -70,7 +70,7 @@ }; static int -flow_dv_tbl_resource_release(struct rte_eth_dev *dev, +flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh, struct mlx5_flow_tbl_resource *tbl); static int @@ -2730,7 +2730,7 @@ struct mlx5_hlist_entry * (void *)&tbl_data->jump, cnt); } else { /* old jump should not make the table ref++. */ - flow_dv_tbl_resource_release(dev, &tbl_data->tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl); MLX5_ASSERT(tbl_data->jump.action); DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++", (void *)&tbl_data->jump, cnt); @@ -7608,6 +7608,7 @@ struct mlx5_hlist_entry * return NULL; } tbl_data->idx = idx; + tbl_data->direction = key.direction; tbl = &tbl_data->tbl; if (key.dummy) return &tbl_data->entry; @@ -7625,6 +7626,13 @@ struct mlx5_hlist_entry * mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); return NULL; } + MKSTR(matcher_name, "%s_%s_%u_matcher_cache", + key.domain ? "FDB" : "NIC", key.direction ? "egress" : "ingress", + key.table_id); + mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh, + flow_dv_matcher_create_cb, + flow_dv_matcher_match_cb, + flow_dv_matcher_remove_cb); rte_atomic32_init(&tbl_data->jump.refcnt); return &tbl_data->entry; } @@ -7684,14 +7692,15 @@ struct mlx5_flow_tbl_resource * MLX5_ASSERT(entry && sh); if (tbl_data->tbl.obj) mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj); + mlx5_cache_list_destroy(&tbl_data->matchers); mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx); } /** * Release a flow table. * - * @param[in] dev - * Pointer to rte_eth_dev structure. + * @param[in] sh + * Pointer to device shared structure. * @param[in] tbl * Table resource to be released. * @@ -7699,11 +7708,9 @@ struct mlx5_flow_tbl_resource * * Returns 0 if table was released, else return 1; */ static int -flow_dv_tbl_resource_release(struct rte_eth_dev *dev, +flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh, struct mlx5_flow_tbl_resource *tbl) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_flow_tbl_data_entry *tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); @@ -7712,6 +7719,63 @@ struct mlx5_flow_tbl_resource * return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry); } +int +flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_matcher *ref = ctx->data; + struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur), + entry); + + return cur->crc != ref->crc || + cur->priority != ref->priority || + memcmp((const void *)cur->mask.buf, + (const void *)ref->mask.buf, ref->mask.size); +} + +struct mlx5_cache_entry * +flow_dv_matcher_create_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_matcher *ref = ctx->data; + struct mlx5_flow_dv_matcher *cache; + struct mlx5dv_flow_matcher_attr dv_attr = { + .type = IBV_FLOW_ATTR_NORMAL, + .match_mask = (void *)&ref->mask, + }; + struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl, + typeof(*tbl), tbl); + int ret; + + cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY); + if (!cache) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot create matcher"); + return NULL; + } + *cache = *ref; + dv_attr.match_criteria_enable = + flow_dv_matcher_enable(cache->mask.buf); + dv_attr.priority = ref->priority; + if (tbl->direction) + dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; + ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj, + &cache->matcher_object); + if (ret) { + mlx5_free(cache); + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot create matcher"); + return NULL; + } + return &cache->entry; +} + /** * Register the flow matcher. * @@ -7731,87 +7795,35 @@ struct mlx5_flow_tbl_resource * */ static int flow_dv_matcher_register(struct rte_eth_dev *dev, - struct mlx5_flow_dv_matcher *matcher, + struct mlx5_flow_dv_matcher *ref, union mlx5_flow_tbl_key *key, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_matcher *cache_matcher; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .match_mask = (void *)&matcher->mask, - }; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher *cache; struct mlx5_flow_tbl_resource *tbl; struct mlx5_flow_tbl_data_entry *tbl_data; - int ret; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = ref, + }; tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, key->domain, 0, error); if (!tbl) return -rte_errno; /* No need to refill the error info */ tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); - /* Lookup from cache. */ - LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) { - if (matcher->crc == cache_matcher->crc && - matcher->priority == cache_matcher->priority && - !memcmp((const void *)matcher->mask.buf, - (const void *)cache_matcher->mask.buf, - cache_matcher->mask.size)) { - DRV_LOG(DEBUG, - "%s group %u priority %hd use %s " - "matcher %p: refcnt %d++", - key->domain ? "FDB" : "NIC", key->table_id, - cache_matcher->priority, - key->direction ? "tx" : "rx", - (void *)cache_matcher, - rte_atomic32_read(&cache_matcher->refcnt)); - rte_atomic32_inc(&cache_matcher->refcnt); - dev_flow->handle->dvh.matcher = cache_matcher; - /* old matcher should not make the table ref++. */ - flow_dv_tbl_resource_release(dev, tbl); - return 0; - } - } - /* Register new matcher. */ - cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0, - SOCKET_ID_ANY); - if (!cache_matcher) { - flow_dv_tbl_resource_release(dev, tbl); + ref->tbl = tbl; + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate matcher memory"); + "cannot allocate ref memory"); } - *cache_matcher = *matcher; - dv_attr.match_criteria_enable = - flow_dv_matcher_enable(cache_matcher->mask.buf); - dv_attr.priority = matcher->priority; - if (key->direction) - dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj, - &cache_matcher->matcher_object); - if (ret) { - mlx5_free(cache_matcher); -#ifdef HAVE_MLX5DV_DR - flow_dv_tbl_resource_release(dev, tbl); -#endif - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create matcher"); - } - /* Save the table information */ - cache_matcher->tbl = tbl; - rte_atomic32_init(&cache_matcher->refcnt); - /* only matcher ref++, table ref++ already done above in get API. */ - rte_atomic32_inc(&cache_matcher->refcnt); - LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); - dev_flow->handle->dvh.matcher = cache_matcher; - DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", - key->domain ? "FDB" : "NIC", key->table_id, - cache_matcher->priority, - key->direction ? "tx" : "rx", (void *)cache_matcher, - rte_atomic32_read(&cache_matcher->refcnt)); + cache = container_of(entry, typeof(*cache), entry); + dev_flow->handle->dvh.matcher = cache; return 0; } @@ -8485,7 +8497,7 @@ struct mlx5_hlist_entry * "cannot create jump action."); if (flow_dv_jump_tbl_resource_register (dev, tbl, dev_flow, error)) { - flow_dv_tbl_resource_release(dev, tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); return rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_ACTION, @@ -9055,6 +9067,19 @@ struct mlx5_hlist_entry * return -rte_errno; } +void +flow_dv_matcher_remove_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache), + entry); + + claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object)); + flow_dv_tbl_resource_release(sh, cache->tbl); + mlx5_free(cache); +} + /** * Release the flow matcher. * @@ -9067,27 +9092,15 @@ struct mlx5_hlist_entry * * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_matcher_release(struct rte_eth_dev *dev, +flow_dv_matcher_release(struct rte_eth_dev *dev __rte_unused, struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher; + struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl, + typeof(*tbl), tbl); MLX5_ASSERT(matcher->matcher_object); - DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", - dev->data->port_id, (void *)matcher, - rte_atomic32_read(&matcher->refcnt)); - if (rte_atomic32_dec_and_test(&matcher->refcnt)) { - claim_zero(mlx5_flow_os_destroy_flow_matcher - (matcher->matcher_object)); - LIST_REMOVE(matcher, next); - /* table ref-- in release interface. */ - flow_dv_tbl_resource_release(dev, matcher->tbl); - mlx5_free(matcher); - DRV_LOG(DEBUG, "port %u matcher %p: removed", - dev->data->port_id, (void *)matcher); - return 0; - } - return 1; + return mlx5_cache_unregister(&tbl->matchers, &matcher->entry); } /** @@ -9169,7 +9182,7 @@ struct mlx5_hlist_entry * claim_zero(mlx5_flow_os_destroy_flow_action (cache_resource->action)); /* jump action memory free is inside the table release. */ - flow_dv_tbl_resource_release(dev, &tbl_data->tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl); DRV_LOG(DEBUG, "jump table resource %p: removed", (void *)cache_resource); return 0; @@ -9580,9 +9593,9 @@ struct mlx5_hlist_entry * claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->egress.any_matcher)); if (mtd->egress.tbl) - flow_dv_tbl_resource_release(dev, mtd->egress.tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) - flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); if (mtd->ingress.color_matcher) claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->ingress.color_matcher)); @@ -9590,9 +9603,10 @@ struct mlx5_hlist_entry * claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) - flow_dv_tbl_resource_release(dev, mtd->ingress.tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) - flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), + mtd->ingress.sfx_tbl); if (mtd->transfer.color_matcher) claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->transfer.color_matcher)); @@ -9600,9 +9614,10 @@ struct mlx5_hlist_entry * claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) - flow_dv_tbl_resource_release(dev, mtd->transfer.tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) - flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), + mtd->transfer.sfx_tbl); if (mtd->drop_actn) claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd);