From patchwork Mon Apr 13 01:11:40 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68227 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 53C8EA0577; Mon, 13 Apr 2020 03:12:05 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 0DEF82B86; Mon, 13 Apr 2020 03:11:59 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id B28DF1B53 for ; Mon, 13 Apr 2020 03:11:57 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:40 +0800 Message-Id: <1586740309-449310-2-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 01/10] net/mlx5: add indexed memory pool X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5_utils.c | 261 ++++++++++++++++++++++++++++++++++++++++++ drivers/net/mlx5/mlx5_utils.h | 229 ++++++++++++++++++++++++++++++++++++ 2 files changed, 490 insertions(+) diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c index 4b4fc3c..4cab7f0 100644 --- a/drivers/net/mlx5/mlx5_utils.c +++ b/drivers/net/mlx5/mlx5_utils.c @@ -117,3 +117,264 @@ struct mlx5_hlist_entry * } rte_free(h); } + +static inline void +mlx5_ipool_lock(struct mlx5_indexed_pool *pool) +{ + if (pool->cfg.need_lock) + rte_spinlock_lock(&pool->lock); +} + +static inline void +mlx5_ipool_unlock(struct mlx5_indexed_pool *pool) +{ + if (pool->cfg.need_lock) + rte_spinlock_unlock(&pool->lock); +} + +struct mlx5_indexed_pool * +mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg) +{ + struct mlx5_indexed_pool *pool; + + if (!cfg || !cfg->size || (!cfg->malloc ^ !cfg->free) || + (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) || + ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32)))) + return NULL; + pool = rte_zmalloc("mlx5_ipool", sizeof(*pool), RTE_CACHE_LINE_SIZE); + if (!pool) + return NULL; + pool->cfg = *cfg; + if (!pool->cfg.trunk_size) + pool->cfg.trunk_size = MLX5_IPOOL_DEFAULT_TRUNK_SIZE; + if (!cfg->malloc && !cfg->free) { + pool->cfg.malloc = rte_malloc_socket; + pool->cfg.free = rte_free; + } + pool->free_list = TRUNK_INVALID; + if (pool->cfg.need_lock) + rte_spinlock_init(&pool->lock); + return pool; +} + +static int +mlx5_ipool_grow(struct mlx5_indexed_pool *pool) +{ + struct mlx5_indexed_trunk *trunk; + struct mlx5_indexed_trunk **trunk_tmp; + struct mlx5_indexed_trunk **p; + size_t trunk_size = 0; + size_t bmp_size; + uint32_t idx; + + if (pool->n_trunk_valid == TRUNK_MAX_IDX) + return -ENOMEM; + if (pool->n_trunk_valid == pool->n_trunk) { + /* No free trunk flags, expand trunk list. */ + int n_grow = pool->n_trunk_valid ? pool->n_trunk : + RTE_CACHE_LINE_SIZE / sizeof(void *); + + p = pool->cfg.malloc(pool->cfg.type, + (pool->n_trunk_valid + n_grow) * + sizeof(struct mlx5_indexed_trunk *), + RTE_CACHE_LINE_SIZE, rte_socket_id()); + if (!p) + return -ENOMEM; + if (pool->trunks) + memcpy(p, pool->trunks, pool->n_trunk_valid * + sizeof(struct mlx5_indexed_trunk *)); + memset(RTE_PTR_ADD(p, pool->n_trunk_valid * sizeof(void *)), 0, + n_grow * sizeof(void *)); + trunk_tmp = pool->trunks; + pool->trunks = p; + if (trunk_tmp) + pool->cfg.free(pool->trunks); + pool->n_trunk += n_grow; + } + idx = pool->n_trunk_valid; + trunk_size += sizeof(*trunk); + bmp_size = rte_bitmap_get_memory_footprint(pool->cfg.trunk_size); + trunk_size += pool->cfg.trunk_size * pool->cfg.size + bmp_size; + trunk = pool->cfg.malloc(pool->cfg.type, trunk_size, + RTE_CACHE_LINE_SIZE, rte_socket_id()); + if (!trunk) + return -ENOMEM; + pool->trunks[idx] = trunk; + trunk->idx = idx; + trunk->free = pool->cfg.trunk_size; + trunk->prev = TRUNK_INVALID; + trunk->next = TRUNK_INVALID; + MLX5_ASSERT(pool->free_list == TRUNK_INVALID); + pool->free_list = idx; + /* Mark all entries as available. */ + trunk->bmp = rte_bitmap_init_with_all_set(pool->cfg.trunk_size, + &trunk->data[pool->cfg.trunk_size * pool->cfg.size], + bmp_size); + pool->n_trunk_valid++; +#ifdef POOL_DEBUG + pool->trunk_new++; + pool->trunk_avail++; +#endif + return 0; +} + +void * +mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx) +{ + struct mlx5_indexed_trunk *trunk; + uint64_t slab = 0; + uint32_t iidx = 0; + void *p; + + mlx5_ipool_lock(pool); + if (pool->free_list == TRUNK_INVALID) { + /* If no available trunks, grow new. */ + if (mlx5_ipool_grow(pool)) { + mlx5_ipool_unlock(pool); + return NULL; + } + } + MLX5_ASSERT(pool->free_list != TRUNK_INVALID); + trunk = pool->trunks[pool->free_list]; + MLX5_ASSERT(trunk->free); + if (!rte_bitmap_scan(trunk->bmp, &iidx, &slab)) { + mlx5_ipool_unlock(pool); + return NULL; + } + MLX5_ASSERT(slab); + iidx += __builtin_ctzll(slab); + MLX5_ASSERT(iidx != UINT32_MAX); + MLX5_ASSERT(iidx < pool->cfg.trunk_size); + rte_bitmap_clear(trunk->bmp, iidx); + p = &trunk->data[iidx * pool->cfg.size]; + iidx += trunk->idx * pool->cfg.trunk_size; + iidx += 1; /* non-zero index. */ + trunk->free--; +#ifdef POOL_DEBUG + pool->n_entry++; +#endif + if (!trunk->free) { + /* Full trunk will be removed from free list in imalloc. */ + MLX5_ASSERT(pool->free_list == trunk->idx); + pool->free_list = trunk->next; + if (trunk->next != TRUNK_INVALID) + pool->trunks[trunk->next]->prev = TRUNK_INVALID; + trunk->prev = TRUNK_INVALID; + trunk->next = TRUNK_INVALID; +#ifdef POOL_DEBUG + pool->trunk_empty++; + pool->trunk_avail--; +#endif + } + *idx = iidx; + mlx5_ipool_unlock(pool); + return p; +} + +void * +mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx) +{ + void *entry = mlx5_ipool_malloc(pool, idx); + + if (entry) + memset(entry, 0, pool->cfg.size); + return entry; +} + +void +mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx) +{ + struct mlx5_indexed_trunk *trunk; + uint32_t trunk_idx; + + if (!idx) + return; + idx -= 1; + mlx5_ipool_lock(pool); + trunk_idx = idx / pool->cfg.trunk_size; + if (trunk_idx >= pool->n_trunk_valid) + goto out; + trunk = pool->trunks[trunk_idx]; + if (!trunk || trunk_idx != trunk->idx || + rte_bitmap_get(trunk->bmp, idx % pool->cfg.trunk_size)) + goto out; + rte_bitmap_set(trunk->bmp, idx % pool->cfg.trunk_size); + trunk->free++; + if (trunk->free == 1) { + /* Put into free trunk list head. */ + MLX5_ASSERT(pool->free_list != trunk->idx); + trunk->next = pool->free_list; + trunk->prev = TRUNK_INVALID; + if (pool->free_list != TRUNK_INVALID) + pool->trunks[pool->free_list]->prev = trunk->idx; + pool->free_list = trunk->idx; +#ifdef POOL_DEBUG + pool->trunk_empty--; + pool->trunk_avail++; +#endif + } +#ifdef POOL_DEBUG + pool->n_entry--; +#endif +out: + mlx5_ipool_unlock(pool); +} + +void * +mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx) +{ + struct mlx5_indexed_trunk *trunk; + void *p = NULL; + uint32_t trunk_idx; + + if (!idx) + return NULL; + idx -= 1; + mlx5_ipool_lock(pool); + trunk_idx = idx / pool->cfg.trunk_size; + if (trunk_idx >= pool->n_trunk_valid) + goto out; + trunk = pool->trunks[trunk_idx]; + if (!trunk || trunk_idx != trunk->idx || + rte_bitmap_get(trunk->bmp, idx % pool->cfg.trunk_size)) + goto out; + p = &trunk->data[(idx % pool->cfg.trunk_size) * pool->cfg.size]; +out: + mlx5_ipool_unlock(pool); + return p; +} + +int +mlx5_ipool_destroy(struct mlx5_indexed_pool *pool) +{ + struct mlx5_indexed_trunk **trunks; + uint32_t i; + + MLX5_ASSERT(pool); + mlx5_ipool_lock(pool); + trunks = pool->trunks; + for (i = 0; i < pool->n_trunk; i++) { + if (trunks[i]) + pool->cfg.free(trunks[i]); + } + if (!pool->trunks) + pool->cfg.free(pool->trunks); + mlx5_ipool_unlock(pool); + rte_free(pool); + return 0; +} + +void +mlx5_ipool_dump(struct mlx5_indexed_pool *pool) +{ + printf("Pool %s entry size %u, trunks %u, %d entry per trunk, " + "total: %d\n", + pool->cfg.type, pool->cfg.size, pool->n_trunk_valid, + pool->cfg.trunk_size, pool->n_trunk_valid); +#ifdef POOL_DEBUG + printf("Pool %s entry %ld, trunk alloc %ld, empty: %ld, " + "available %ld free %ld\n", + pool->cfg.type, pool->n_entry, pool->trunk_new, + pool->trunk_empty, pool->trunk_avail, pool->trunk_free); +#endif +} diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h index 8f305c3..e404a5c 100644 --- a/drivers/net/mlx5/mlx5_utils.h +++ b/drivers/net/mlx5/mlx5_utils.h @@ -12,6 +12,10 @@ #include #include +#include +#include +#include + #include #include "mlx5_defs.h" @@ -60,6 +64,60 @@ (((val) & (from)) / ((from) / (to))) : \ (((val) & (from)) * ((to) / (from)))) +/* + * The indexed memory entry index is made up of trunk index and offset of + * the entry in the trunk. Since the entry index is 32 bits, in case user + * prefers to have small trunks, user can change the macro below to a big + * number which helps the pool contains more trunks with lots of entries + * allocated. + */ +#define TRUNK_IDX_BITS 16 +#define TRUNK_MAX_IDX ((1 << TRUNK_IDX_BITS) - 1) +#define TRUNK_INVALID TRUNK_MAX_IDX +#define MLX5_IPOOL_DEFAULT_TRUNK_SIZE (1 << (28 - TRUNK_IDX_BITS)) +#ifdef RTE_LIBRTE_MLX5_DEBUG +#define POOL_DEBUG 1 +#endif + +struct mlx5_indexed_pool_config { + uint32_t size; /* Pool entry size. */ + uint32_t trunk_size; + /* Trunk entry number. Must be power of 2. */ + uint32_t need_lock; + /* Lock is needed for multiple thread usage. */ + const char *type; /* Memory allocate type name. */ + void *(*malloc)(const char *type, size_t size, unsigned int align, + int socket); + /* User defined memory allocator. */ + void (*free)(void *addr); /* User defined memory release. */ +}; + +struct mlx5_indexed_trunk { + uint32_t idx; /* Trunk id. */ + uint32_t prev; /* Previous free trunk in free list. */ + uint32_t next; /* Next free trunk in free list. */ + uint32_t free; /* Free entries available */ + struct rte_bitmap *bmp; + uint8_t data[] __rte_cache_min_aligned; /* Entry data start. */ +}; + +struct mlx5_indexed_pool { + struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */ + rte_spinlock_t lock; /* Pool lock for multiple thread usage. */ + uint32_t n_trunk_valid; /* Trunks allocated. */ + uint32_t n_trunk; /* Trunk pointer array size. */ + /* Dim of trunk pointer array. */ + struct mlx5_indexed_trunk **trunks; + uint32_t free_list; /* Index to first free trunk. */ +#ifdef POOL_DEBUG + int64_t n_entry; + int64_t trunk_new; + int64_t trunk_avail; + int64_t trunk_empty; + int64_t trunk_free; +#endif +}; + /** * Return logarithm of the nearest power of two above input value. * @@ -183,4 +241,175 @@ void mlx5_hlist_remove(struct mlx5_hlist *h __rte_unused, void mlx5_hlist_destroy(struct mlx5_hlist *h, mlx5_hlist_destroy_callback_fn cb, void *ctx); +/** + * This function allocates non-initialized memory entry from pool. + * In NUMA systems, the memory entry allocated resides on the same + * NUMA socket as the core that calls this function. + * + * Memory entry is allocated from memory trunk, no alignment. + * + * @param pool + * Pointer to indexed memory entry pool. + * No initialization required. + * @param[out] idx + * Pointer to memory to save allocated index. + * Memory index always positive value. + * @return + * - Pointer to the allocated memory entry. + * - NULL on error. Not enough memory, or invalid arguments. + */ +void *mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx); + +/** + * This function allocates zero initialized memory entry from pool. + * In NUMA systems, the memory entry allocated resides on the same + * NUMA socket as the core that calls this function. + * + * Memory entry is allocated from memory trunk, no alignment. + * + * @param pool + * Pointer to indexed memory pool. + * No initialization required. + * @param[out] idx + * Pointer to memory to save allocated index. + * Memory index always positive value. + * @return + * - Pointer to the allocated memory entry . + * - NULL on error. Not enough memory, or invalid arguments. + */ +void *mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx); + +/** + * This function frees indexed memory entry to pool. + * Caller has to make sure that the index is allocated from same pool. + * + * @param pool + * Pointer to indexed memory pool. + * @param idx + * Allocated memory entry index. + */ +void mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx); + +/** + * This function returns pointer of indexed memory entry from index. + * Caller has to make sure that the index is valid, and allocated + * from same pool. + * + * @param pool + * Pointer to indexed memory pool. + * @param idx + * Allocated memory index. + * @return + * - Pointer to indexed memory entry. + */ +void *mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx); + +/** + * This function creates indexed memory pool. + * Caller has to configure the configuration accordingly. + * + * @param pool + * Pointer to indexed memory pool. + * @param cfg + * Allocated memory index. + */ +struct mlx5_indexed_pool * +mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg); + +/** + * This function releases all resources of pool. + * Caller has to make sure that all indexes and memories allocated + * from this pool not referenced anymore. + * + * @param pool + * Pointer to indexed memory pool. + * @return + * - non-zero value on error. + * - 0 on success. + */ +int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool); + +/** + * This function dumps debug info of pool. + * + * @param pool + * Pointer to indexed memory pool. + */ +void mlx5_ipool_dump(struct mlx5_indexed_pool *pool); + +/* + * Macros for linked list based on indexed memory. + * Example data structure: + * struct Foo { + * ILIST_ENTRY(uint16_t) next; + * ... + * } + * + */ +#define ILIST_ENTRY(type) \ +struct { \ + type prev; /* Index of previous element. */ \ + type next; /* Index of next element. */ \ +} + +#define ILIST_INSERT(pool, head, idx, elem, field) \ + do { \ + typeof(elem) peer; \ + MLX5_ASSERT((elem) && (idx)); \ + (elem)->field.next = *(head); \ + (elem)->field.prev = 0; \ + if (*(head)) { \ + (peer) = mlx5_ipool_get(pool, *(head)); \ + if (peer) \ + (peer)->field.prev = (idx); \ + } \ + *(head) = (idx); \ + } while (0) + +#define ILIST_REMOVE(pool, head, idx, elem, field) \ + do { \ + typeof(elem) peer; \ + MLX5_ASSERT(elem); \ + MLX5_ASSERT(head); \ + if ((elem)->field.prev) { \ + (peer) = mlx5_ipool_get \ + (pool, (elem)->field.prev); \ + if (peer) \ + (peer)->field.next = (elem)->field.next;\ + } \ + if ((elem)->field.next) { \ + (peer) = mlx5_ipool_get \ + (pool, (elem)->field.next); \ + if (peer) \ + (peer)->field.prev = (elem)->field.prev;\ + } \ + if (*(head) == (idx)) \ + *(head) = (elem)->field.next; \ + } while (0) + +#define ILIST_FOREACH(pool, head, idx, elem, field) \ + for ((idx) = (head), (elem) = \ + (idx) ? mlx5_ipool_get(pool, (idx)) : NULL; (elem); \ + idx = (elem)->field.next, (elem) = \ + (idx) ? mlx5_ipool_get(pool, idx) : NULL) + +/* Single index list. */ +#define SILIST_ENTRY(type) \ +struct { \ + type next; /* Index of next element. */ \ +} + +#define SILIST_INSERT(head, idx, elem, field) \ + do { \ + MLX5_ASSERT((elem) && (idx)); \ + (elem)->field.next = *(head); \ + *(head) = (idx); \ + } while (0) + +#define SILIST_FOREACH(pool, head, idx, elem, field) \ + for ((idx) = (head), (elem) = \ + (idx) ? mlx5_ipool_get(pool, (idx)) : NULL; (elem); \ + idx = (elem)->field.next, (elem) = \ + (idx) ? mlx5_ipool_get(pool, idx) : NULL) + #endif /* RTE_PMD_MLX5_UTILS_H_ */ From patchwork Mon Apr 13 01:11:41 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68228 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 40FF2A0577; Mon, 13 Apr 2020 03:12:19 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 026DE2C54; Mon, 13 Apr 2020 03:12:02 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id 965D72BAB for ; Mon, 13 Apr 2020 03:11:59 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:41 +0800 Message-Id: <1586740309-449310-3-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 02/10] net/mlx5: add trunk dynamic grow for indexed pool X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5_utils.c | 105 +++++++++++++++++++++++++++++++++++------- drivers/net/mlx5/mlx5_utils.h | 23 +++++++-- 2 files changed, 108 insertions(+), 20 deletions(-) diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c index 4cab7f0..e63921d 100644 --- a/drivers/net/mlx5/mlx5_utils.c +++ b/drivers/net/mlx5/mlx5_utils.c @@ -132,16 +132,69 @@ struct mlx5_hlist_entry * rte_spinlock_unlock(&pool->lock); } +static inline uint32_t +mlx5_trunk_idx_get(struct mlx5_indexed_pool *pool, uint32_t entry_idx) +{ + struct mlx5_indexed_pool_config *cfg = &pool->cfg; + uint32_t trunk_idx = 0; + uint32_t i; + + if (!cfg->grow_trunk) + return entry_idx / cfg->trunk_size; + if (entry_idx >= pool->grow_tbl[cfg->grow_trunk - 1]) { + trunk_idx = (entry_idx - pool->grow_tbl[cfg->grow_trunk - 1]) / + (cfg->trunk_size << (cfg->grow_shift * + cfg->grow_trunk)) + cfg->grow_trunk; + } else { + for (i = 0; i < cfg->grow_trunk; i++) { + if (entry_idx < pool->grow_tbl[i]) + break; + } + trunk_idx = i; + } + return trunk_idx; +} + +static inline uint32_t +mlx5_trunk_size_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx) +{ + struct mlx5_indexed_pool_config *cfg = &pool->cfg; + + return cfg->trunk_size << (cfg->grow_shift * + (trunk_idx > cfg->grow_trunk ? cfg->grow_trunk : trunk_idx)); +} + +static inline uint32_t +mlx5_trunk_idx_offset_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx) +{ + struct mlx5_indexed_pool_config *cfg = &pool->cfg; + uint32_t offset = 0; + + if (!trunk_idx) + return 0; + if (!cfg->grow_trunk) + return cfg->trunk_size * trunk_idx; + if (trunk_idx < cfg->grow_trunk) + offset = pool->grow_tbl[trunk_idx - 1]; + else + offset = pool->grow_tbl[cfg->grow_trunk - 1] + + (cfg->trunk_size << (cfg->grow_shift * + cfg->grow_trunk)) * (trunk_idx - cfg->grow_trunk); + return offset; +} + struct mlx5_indexed_pool * mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg) { struct mlx5_indexed_pool *pool; + uint32_t i; if (!cfg || !cfg->size || (!cfg->malloc ^ !cfg->free) || (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) || ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32)))) return NULL; - pool = rte_zmalloc("mlx5_ipool", sizeof(*pool), RTE_CACHE_LINE_SIZE); + pool = rte_zmalloc("mlx5_ipool", sizeof(*pool) + cfg->grow_trunk * + sizeof(pool->grow_tbl[0]), RTE_CACHE_LINE_SIZE); if (!pool) return NULL; pool->cfg = *cfg; @@ -154,6 +207,15 @@ struct mlx5_indexed_pool * pool->free_list = TRUNK_INVALID; if (pool->cfg.need_lock) rte_spinlock_init(&pool->lock); + /* + * Initialize the dynamic grow trunk size lookup table to have a quick + * lookup for the trunk entry index offset. + */ + for (i = 0; i < cfg->grow_trunk; i++) { + pool->grow_tbl[i] = cfg->trunk_size << (cfg->grow_shift * i); + if (i > 0) + pool->grow_tbl[i] += pool->grow_tbl[i - 1]; + } return pool; } @@ -164,6 +226,7 @@ struct mlx5_indexed_pool * struct mlx5_indexed_trunk **trunk_tmp; struct mlx5_indexed_trunk **p; size_t trunk_size = 0; + size_t data_size; size_t bmp_size; uint32_t idx; @@ -193,23 +256,23 @@ struct mlx5_indexed_pool * } idx = pool->n_trunk_valid; trunk_size += sizeof(*trunk); - bmp_size = rte_bitmap_get_memory_footprint(pool->cfg.trunk_size); - trunk_size += pool->cfg.trunk_size * pool->cfg.size + bmp_size; + data_size = mlx5_trunk_size_get(pool, idx); + bmp_size = rte_bitmap_get_memory_footprint(data_size); + trunk_size += data_size * pool->cfg.size + bmp_size; trunk = pool->cfg.malloc(pool->cfg.type, trunk_size, RTE_CACHE_LINE_SIZE, rte_socket_id()); if (!trunk) return -ENOMEM; pool->trunks[idx] = trunk; trunk->idx = idx; - trunk->free = pool->cfg.trunk_size; + trunk->free = data_size; trunk->prev = TRUNK_INVALID; trunk->next = TRUNK_INVALID; MLX5_ASSERT(pool->free_list == TRUNK_INVALID); pool->free_list = idx; /* Mark all entries as available. */ - trunk->bmp = rte_bitmap_init_with_all_set(pool->cfg.trunk_size, - &trunk->data[pool->cfg.trunk_size * pool->cfg.size], - bmp_size); + trunk->bmp = rte_bitmap_init_with_all_set(data_size, + &trunk->data[data_size * pool->cfg.size], bmp_size); pool->n_trunk_valid++; #ifdef POOL_DEBUG pool->trunk_new++; @@ -244,10 +307,10 @@ struct mlx5_indexed_pool * MLX5_ASSERT(slab); iidx += __builtin_ctzll(slab); MLX5_ASSERT(iidx != UINT32_MAX); - MLX5_ASSERT(iidx < pool->cfg.trunk_size); + MLX5_ASSERT(iidx < mlx5_trunk_size_get(pool, trunk->idx)); rte_bitmap_clear(trunk->bmp, iidx); p = &trunk->data[iidx * pool->cfg.size]; - iidx += trunk->idx * pool->cfg.trunk_size; + iidx += mlx5_trunk_idx_offset_get(pool, trunk->idx); iidx += 1; /* non-zero index. */ trunk->free--; #ifdef POOL_DEBUG @@ -286,19 +349,23 @@ struct mlx5_indexed_pool * { struct mlx5_indexed_trunk *trunk; uint32_t trunk_idx; + uint32_t entry_idx; if (!idx) return; idx -= 1; mlx5_ipool_lock(pool); - trunk_idx = idx / pool->cfg.trunk_size; + trunk_idx = mlx5_trunk_idx_get(pool, idx); if (trunk_idx >= pool->n_trunk_valid) goto out; trunk = pool->trunks[trunk_idx]; - if (!trunk || trunk_idx != trunk->idx || - rte_bitmap_get(trunk->bmp, idx % pool->cfg.trunk_size)) + if (!trunk) + goto out; + entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx); + if (trunk_idx != trunk->idx || + rte_bitmap_get(trunk->bmp, entry_idx)) goto out; - rte_bitmap_set(trunk->bmp, idx % pool->cfg.trunk_size); + rte_bitmap_set(trunk->bmp, entry_idx); trunk->free++; if (trunk->free == 1) { /* Put into free trunk list head. */ @@ -326,19 +393,23 @@ struct mlx5_indexed_pool * struct mlx5_indexed_trunk *trunk; void *p = NULL; uint32_t trunk_idx; + uint32_t entry_idx; if (!idx) return NULL; idx -= 1; mlx5_ipool_lock(pool); - trunk_idx = idx / pool->cfg.trunk_size; + trunk_idx = mlx5_trunk_idx_get(pool, idx); if (trunk_idx >= pool->n_trunk_valid) goto out; trunk = pool->trunks[trunk_idx]; - if (!trunk || trunk_idx != trunk->idx || - rte_bitmap_get(trunk->bmp, idx % pool->cfg.trunk_size)) + if (!trunk) + goto out; + entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx); + if (trunk_idx != trunk->idx || + rte_bitmap_get(trunk->bmp, entry_idx)) goto out; - p = &trunk->data[(idx % pool->cfg.trunk_size) * pool->cfg.size]; + p = &trunk->data[entry_idx * pool->cfg.size]; out: mlx5_ipool_unlock(pool); return p; diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h index e404a5c..af96a87 100644 --- a/drivers/net/mlx5/mlx5_utils.h +++ b/drivers/net/mlx5/mlx5_utils.h @@ -81,9 +81,25 @@ struct mlx5_indexed_pool_config { uint32_t size; /* Pool entry size. */ - uint32_t trunk_size; - /* Trunk entry number. Must be power of 2. */ - uint32_t need_lock; + uint32_t trunk_size:22; + /* + * Trunk entry number. Must be power of 2. It can be increased + * if trunk_grow enable. The trunk entry number increases with + * left shift grow_shift. Trunks with index are after grow_trunk + * will keep the entry number same with the last grow trunk. + */ + uint32_t grow_trunk:4; + /* + * Trunks with entry number increase in the pool. Set it to 0 + * to make the pool works as trunk entry fixed pool. It works + * only if grow_shift is not 0. + */ + uint32_t grow_shift:4; + /* + * Trunk entry number increase shift value, stop after grow_trunk. + * It works only if grow_trunk is not 0. + */ + uint32_t need_lock:1; /* Lock is needed for multiple thread usage. */ const char *type; /* Memory allocate type name. */ void *(*malloc)(const char *type, size_t size, unsigned int align, @@ -116,6 +132,7 @@ struct mlx5_indexed_pool { int64_t trunk_empty; int64_t trunk_free; #endif + uint32_t grow_tbl[]; /* Save the index offset for the grow trunks. */ }; /** From patchwork Mon Apr 13 01:11:42 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68229 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 745C4A0577; Mon, 13 Apr 2020 03:12:28 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 46C542BFA; Mon, 13 Apr 2020 03:12:04 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id 7CC112BE9 for ; Mon, 13 Apr 2020 03:12:01 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:42 +0800 Message-Id: <1586740309-449310-4-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 03/10] net/mlx5: add trunk release for indexed pool X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" While entries are fully freed in trunk, it means the trunk is free now. User may prefer the free trunk memory can be reclaimed. Add the trunk release memory option for indexed pool in this case. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5_utils.c | 37 +++++++++++++++++++++++++++++++++---- drivers/net/mlx5/mlx5_utils.h | 1 + 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c index e63921d..7683167 100644 --- a/drivers/net/mlx5/mlx5_utils.c +++ b/drivers/net/mlx5/mlx5_utils.c @@ -254,7 +254,14 @@ struct mlx5_indexed_pool * pool->cfg.free(pool->trunks); pool->n_trunk += n_grow; } - idx = pool->n_trunk_valid; + if (!pool->cfg.release_mem_en) { + idx = pool->n_trunk_valid; + } else { + /* Find the first available slot in trunk list */ + for (idx = 0; idx < pool->n_trunk; idx++) + if (pool->trunks[idx] == NULL) + break; + } trunk_size += sizeof(*trunk); data_size = mlx5_trunk_size_get(pool, idx); bmp_size = rte_bitmap_get_memory_footprint(data_size); @@ -356,7 +363,8 @@ struct mlx5_indexed_pool * idx -= 1; mlx5_ipool_lock(pool); trunk_idx = mlx5_trunk_idx_get(pool, idx); - if (trunk_idx >= pool->n_trunk_valid) + if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) || + (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk)) goto out; trunk = pool->trunks[trunk_idx]; if (!trunk) @@ -367,7 +375,27 @@ struct mlx5_indexed_pool * goto out; rte_bitmap_set(trunk->bmp, entry_idx); trunk->free++; - if (trunk->free == 1) { + if (pool->cfg.release_mem_en && trunk->free == mlx5_trunk_size_get + (pool, trunk->idx)) { + if (pool->free_list == trunk->idx) + pool->free_list = trunk->next; + if (trunk->next != TRUNK_INVALID) + pool->trunks[trunk->next]->prev = trunk->prev; + if (trunk->prev != TRUNK_INVALID) + pool->trunks[trunk->prev]->next = trunk->next; + pool->cfg.free(trunk); + pool->trunks[trunk_idx] = NULL; + pool->n_trunk_valid--; +#ifdef POOL_DEBUG + pool->trunk_avail--; + pool->trunk_free++; +#endif + if (pool->n_trunk_valid == 0) { + pool->cfg.free(pool->trunks); + pool->trunks = NULL; + pool->n_trunk = 0; + } + } else if (trunk->free == 1) { /* Put into free trunk list head. */ MLX5_ASSERT(pool->free_list != trunk->idx); trunk->next = pool->free_list; @@ -400,7 +428,8 @@ struct mlx5_indexed_pool * idx -= 1; mlx5_ipool_lock(pool); trunk_idx = mlx5_trunk_idx_get(pool, idx); - if (trunk_idx >= pool->n_trunk_valid) + if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) || + (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk)) goto out; trunk = pool->trunks[trunk_idx]; if (!trunk) diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h index af96a87..1051b12 100644 --- a/drivers/net/mlx5/mlx5_utils.h +++ b/drivers/net/mlx5/mlx5_utils.h @@ -101,6 +101,7 @@ struct mlx5_indexed_pool_config { */ uint32_t need_lock:1; /* Lock is needed for multiple thread usage. */ + uint32_t release_mem_en:1; /* Rlease trunk when it is free. */ const char *type; /* Memory allocate type name. */ void *(*malloc)(const char *type, size_t size, unsigned int align, int socket); From patchwork Mon Apr 13 01:11:43 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68230 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 2FDC2A0577; Mon, 13 Apr 2020 03:12:37 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id AB7D71BE9D; Mon, 13 Apr 2020 03:12:05 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id 713A52BFA for ; Mon, 13 Apr 2020 03:12:03 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:43 +0800 Message-Id: <1586740309-449310-5-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 04/10] net/mlx5: convert encap/decap resource to indexed X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit converts the flow encap/decap resource to indexed. Using the uint32_t index instead of pointer saves 4 bytes memory for the flow handle. For millions flows, it will save several MBytes of memory. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.c | 47 ++++++++++++++++++++++++++++++++++++++++ drivers/net/mlx5/mlx5.h | 9 +++++++- drivers/net/mlx5/mlx5_flow.h | 8 ++++--- drivers/net/mlx5/mlx5_flow_dv.c | 48 +++++++++++++++++++++++++++-------------- 4 files changed, 92 insertions(+), 20 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 6a11b14..f4c2c14 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -194,6 +194,21 @@ struct mlx5_dev_spawn_data { static LIST_HEAD(, mlx5_ibv_shared) mlx5_ibv_list = LIST_HEAD_INITIALIZER(); static pthread_mutex_t mlx5_ibv_list_mutex = PTHREAD_MUTEX_INITIALIZER; +static struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { + { + .size = sizeof(struct mlx5_flow_dv_encap_decap_resource), + .trunk_size = 64, + .grow_trunk = 3, + .grow_shift = 2, + .need_lock = 0, + .release_mem_en = 1, + .malloc = rte_malloc_socket, + .free = rte_free, + .type = "mlx5_encap_decap_ipool", + }, +}; + + #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 @@ -411,6 +426,36 @@ struct mlx5_flow_id_pool * } /** + * Initialize the flow resources' indexed mempool. + * + * @param[in] sh + * Pointer to mlx5_ibv_shared object. + */ +static void +mlx5_flow_ipool_create(struct mlx5_ibv_shared *sh) +{ + uint8_t i; + + for (i = 0; i < MLX5_IPOOL_MAX; ++i) + sh->ipool[i] = mlx5_ipool_create(&mlx5_ipool_cfg[i]); +} + +/** + * Release the flow resources' indexed mempool. + * + * @param[in] sh + * Pointer to mlx5_ibv_shared object. + */ +static void +mlx5_flow_ipool_destroy(struct mlx5_ibv_shared *sh) +{ + uint8_t i; + + for (i = 0; i < MLX5_IPOOL_MAX; ++i) + mlx5_ipool_destroy(sh->ipool[i]); +} + +/** * Extract pdn of PD object using DV API. * * @param[in] pd @@ -625,6 +670,7 @@ struct mlx5_flow_id_pool * goto error; } mlx5_flow_counters_mng_init(sh); + mlx5_flow_ipool_create(sh); /* Add device to memory callback list. */ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list, @@ -697,6 +743,7 @@ struct mlx5_flow_id_pool * * Only primary process handles async device events. **/ mlx5_flow_counters_mng_close(sh); + mlx5_flow_ipool_destroy(sh); MLX5_ASSERT(!sh->intr_cnt); if (sh->intr_cnt) mlx5_intr_callback_unregister diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 34ab475..a1a7294 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -51,6 +51,11 @@ enum mlx5_mp_req_type { MLX5_MP_REQ_QUEUE_STATE_MODIFY, }; +enum mlx5_ipool_index { + MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */ + MLX5_IPOOL_MAX, +}; + struct mlx5_mp_arg_queue_state_modify { uint8_t is_wq; /* Set if WQ. */ uint16_t queue_id; /* DPDK queue ID. */ @@ -446,7 +451,7 @@ struct mlx5_ibv_shared { /* Direct Rules tables for FDB, NIC TX+RX */ void *esw_drop_action; /* Pointer to DR E-Switch drop action. */ void *pop_vlan_action; /* Pointer to DR pop VLAN action. */ - LIST_HEAD(encap_decap, mlx5_flow_dv_encap_decap_resource) encaps_decaps; + uint32_t encaps_decaps; /* Encap/decap action indexed memory list. */ LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds; struct mlx5_hlist *tag_table; LIST_HEAD(port_id_action_list, mlx5_flow_dv_port_id_action_resource) @@ -454,6 +459,8 @@ struct mlx5_ibv_shared { LIST_HEAD(push_vlan_action_list, mlx5_flow_dv_push_vlan_action_resource) push_vlan_action_list; /* List of push VLAN actions. */ struct mlx5_flow_counter_mng cmng; /* Counters management structure. */ + struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX]; + /* Memory Pool for mlx5 flow resources. */ /* Shared interrupt handler section. */ pthread_mutex_t intr_mutex; /* Interrupt config mutex. */ uint32_t intr_cnt; /* Interrupt handler reference counter. */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 0f0e59d..77d6b2d 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -364,7 +364,7 @@ struct mlx5_flow_dv_matcher { /* Encap/decap resource structure. */ struct mlx5_flow_dv_encap_decap_resource { - LIST_ENTRY(mlx5_flow_dv_encap_decap_resource) next; + ILIST_ENTRY(uint32_t)next; /* Pointer to next element. */ rte_atomic32_t refcnt; /**< Reference counter. */ void *verbs_action; @@ -482,8 +482,8 @@ struct mlx5_flow_rss { struct mlx5_flow_handle_dv { /* Flow DV api: */ struct mlx5_flow_dv_matcher *matcher; /**< Cache to matcher. */ - struct mlx5_flow_dv_encap_decap_resource *encap_decap; - /**< Pointer to encap/decap resource in cache. */ + uint32_t encap_decap; + /**< Index to encap/decap resource in cache. */ struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; /**< Pointer to modify header resource in cache. */ struct mlx5_flow_dv_jump_tbl_resource *jump; @@ -543,6 +543,8 @@ struct mlx5_flow_dv_workspace { uint8_t transfer; /**< 1 if the flow is E-Switch flow. */ int actions_n; /**< number of actions. */ void *actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS]; /**< Action list. */ + struct mlx5_flow_dv_encap_decap_resource *encap_decap; + /**< Pointer to encap/decap resource in cache. */ struct mlx5_flow_dv_match_params value; /**< Holds the value that the packet is compared to. */ }; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index f5d98d2..8275098 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -2437,6 +2437,7 @@ struct field_modify_info modify_tcp[] = { struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_encap_decap_resource *cache_resource; struct mlx5dv_dr_domain *domain; + uint32_t idx = 0; resource->flags = dev_flow->dv.group ? 0 : 1; if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) @@ -2446,7 +2447,8 @@ struct field_modify_info modify_tcp[] = { else domain = sh->tx_domain; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) { + ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx, + cache_resource, next) { if (resource->reformat_type == cache_resource->reformat_type && resource->ft_type == cache_resource->ft_type && resource->flags == cache_resource->flags && @@ -2458,12 +2460,14 @@ struct field_modify_info modify_tcp[] = { (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.encap_decap = cache_resource; + dev_flow->handle->dvh.encap_decap = idx; + dev_flow->dv.encap_decap = cache_resource; return 0; } } /* Register new encap/decap resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], + &dev_flow->handle->dvh.encap_decap); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -2483,8 +2487,9 @@ struct field_modify_info modify_tcp[] = { } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next); - dev_flow->handle->dvh.encap_decap = cache_resource; + ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps, + dev_flow->handle->dvh.encap_decap, cache_resource, next); + dev_flow->dv.encap_decap = cache_resource; DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -3080,6 +3085,7 @@ struct field_modify_info modify_tcp[] = { const struct rte_flow_action_raw_encap *encap_data; struct mlx5_flow_dv_encap_decap_resource res; + memset(&res, 0, sizeof(res)); encap_data = (const struct rte_flow_action_raw_encap *)action->conf; res.size = encap_data->size; memcpy(res.buf, encap_data->data, res.size); @@ -7572,7 +7578,7 @@ struct field_modify_info modify_tcp[] = { error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - handle->dvh.encap_decap->verbs_action; + dev_flow->dv.encap_decap->verbs_action; action_flags |= MLX5_FLOW_ACTION_ENCAP; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -7582,7 +7588,7 @@ struct field_modify_info modify_tcp[] = { error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - handle->dvh.encap_decap->verbs_action; + dev_flow->dv.encap_decap->verbs_action; action_flags |= MLX5_FLOW_ACTION_DECAP; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: @@ -7592,7 +7598,7 @@ struct field_modify_info modify_tcp[] = { (dev, actions, dev_flow, attr, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - handle->dvh.encap_decap->verbs_action; + dev_flow->dv.encap_decap->verbs_action; } else { /* Handle encap without preceding decap. */ if (flow_dv_create_action_l2_encap @@ -7600,7 +7606,7 @@ struct field_modify_info modify_tcp[] = { error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - handle->dvh.encap_decap->verbs_action; + dev_flow->dv.encap_decap->verbs_action; } action_flags |= MLX5_FLOW_ACTION_ENCAP; break; @@ -7612,7 +7618,7 @@ struct field_modify_info modify_tcp[] = { (dev, dev_flow, attr->transfer, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - handle->dvh.encap_decap->verbs_action; + dev_flow->dv.encap_decap->verbs_action; } /* If decap is followed by encap, handle it at encap. */ action_flags |= MLX5_FLOW_ACTION_DECAP; @@ -8166,6 +8172,8 @@ struct field_modify_info modify_tcp[] = { /** * Release an encap/decap resource. * + * @param dev + * Pointer to Ethernet device. * @param handle * Pointer to mlx5_flow_handle. * @@ -8173,11 +8181,17 @@ struct field_modify_info modify_tcp[] = { * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_encap_decap_resource_release(struct mlx5_flow_handle *handle) +flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow_handle *handle) { - struct mlx5_flow_dv_encap_decap_resource *cache_resource = - handle->dvh.encap_decap; + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t idx = handle->dvh.encap_decap; + struct mlx5_flow_dv_encap_decap_resource *cache_resource; + cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], + idx); + if (!cache_resource) + return 0; MLX5_ASSERT(cache_resource->verbs_action); DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", (void *)cache_resource, @@ -8185,8 +8199,10 @@ struct field_modify_info modify_tcp[] = { if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { claim_zero(mlx5_glue->destroy_flow_action (cache_resource->verbs_action)); - LIST_REMOVE(cache_resource, next); - rte_free(cache_resource); + ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], + &priv->sh->encaps_decaps, idx, + cache_resource, next); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx); DRV_LOG(DEBUG, "encap/decap resource %p: removed", (void *)cache_resource); return 0; @@ -8388,7 +8404,7 @@ struct field_modify_info modify_tcp[] = { if (dev_handle->dvh.matcher) flow_dv_matcher_release(dev, dev_handle); if (dev_handle->dvh.encap_decap) - flow_dv_encap_decap_resource_release(dev_handle); + flow_dv_encap_decap_resource_release(dev, dev_handle); if (dev_handle->dvh.modify_hdr) flow_dv_modify_hdr_resource_release(dev_handle); if (dev_handle->dvh.jump) From patchwork Mon Apr 13 01:11:44 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68231 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 2B1E0A0577; Mon, 13 Apr 2020 03:12:46 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 1E9B71BEDC; Mon, 13 Apr 2020 03:12:07 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id 7274B1BE90 for ; Mon, 13 Apr 2020 03:12:05 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:44 +0800 Message-Id: <1586740309-449310-6-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 05/10] net/mlx5: convert push VLAN resource to indexed X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit converts the push VLAN resource to indexed. Using the uint32_t index instead of pointer saves 4 bytes memory for the flow handle. For millions flows, it will save several MBytes of memory. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.c | 11 +++++++++++ drivers/net/mlx5/mlx5.h | 4 ++-- drivers/net/mlx5/mlx5_flow.h | 8 +++++--- drivers/net/mlx5/mlx5_flow_dv.c | 43 +++++++++++++++++++++++++++++------------ 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index f4c2c14..3a26378 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -206,6 +206,17 @@ struct mlx5_dev_spawn_data { .free = rte_free, .type = "mlx5_encap_decap_ipool", }, + { + .size = sizeof(struct mlx5_flow_dv_push_vlan_action_resource), + .trunk_size = 64, + .grow_trunk = 3, + .grow_shift = 2, + .need_lock = 0, + .release_mem_en = 1, + .malloc = rte_malloc_socket, + .free = rte_free, + .type = "mlx5_push_vlan_ipool", + }, }; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index a1a7294..10e9071 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -53,6 +53,7 @@ enum mlx5_mp_req_type { enum mlx5_ipool_index { MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */ + MLX5_IPOOL_PUSH_VLAN, /* Pool for push vlan resource. */ MLX5_IPOOL_MAX, }; @@ -456,8 +457,7 @@ struct mlx5_ibv_shared { struct mlx5_hlist *tag_table; LIST_HEAD(port_id_action_list, mlx5_flow_dv_port_id_action_resource) port_id_action_list; /* List of port ID actions. */ - LIST_HEAD(push_vlan_action_list, mlx5_flow_dv_push_vlan_action_resource) - push_vlan_action_list; /* List of push VLAN actions. */ + uint32_t push_vlan_action_list; /* List of push VLAN actions. */ struct mlx5_flow_counter_mng cmng; /* Counters management structure. */ struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX]; /* Memory Pool for mlx5 flow resources. */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 77d6b2d..8d169fe 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -429,7 +429,7 @@ struct mlx5_flow_dv_port_id_action_resource { /* Push VLAN action resource structure */ struct mlx5_flow_dv_push_vlan_action_resource { - LIST_ENTRY(mlx5_flow_dv_push_vlan_action_resource) next; + ILIST_ENTRY(uint32_t)next; /* Pointer to next element. */ rte_atomic32_t refcnt; /**< Reference counter. */ void *action; /**< Direct verbs action object. */ @@ -492,8 +492,8 @@ struct mlx5_flow_handle_dv { /**< Pointer to port ID action resource. */ struct mlx5_vf_vlan vf_vlan; /**< Structure for VF VLAN workaround. */ - struct mlx5_flow_dv_push_vlan_action_resource *push_vlan_res; - /**< Pointer to push VLAN action resource in cache. */ + uint32_t push_vlan_res; + /**< Index to push VLAN action resource in cache. */ struct mlx5_flow_dv_tag_resource *tag_resource; /**< pointer to the tag action. */ }; @@ -545,6 +545,8 @@ struct mlx5_flow_dv_workspace { void *actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS]; /**< Action list. */ struct mlx5_flow_dv_encap_decap_resource *encap_decap; /**< Pointer to encap/decap resource in cache. */ + struct mlx5_flow_dv_push_vlan_action_resource *push_vlan_res; + /**< Pointer to push VLAN action resource in cache. */ struct mlx5_flow_dv_match_params value; /**< Holds the value that the packet is compared to. */ }; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 8275098..9965374 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -2639,9 +2639,11 @@ struct field_modify_info modify_tcp[] = { struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; struct mlx5dv_dr_domain *domain; + uint32_t idx = 0; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &sh->push_vlan_action_list, next) { + ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN], + sh->push_vlan_action_list, idx, cache_resource, next) { if (resource->vlan_tag == cache_resource->vlan_tag && resource->ft_type == cache_resource->ft_type) { DRV_LOG(DEBUG, "push-VLAN action resource resource %p: " @@ -2649,12 +2651,14 @@ struct field_modify_info modify_tcp[] = { (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.push_vlan_res = cache_resource; + dev_flow->handle->dvh.push_vlan_res = idx; + dev_flow->dv.push_vlan_res = cache_resource; return 0; } } /* Register new push_vlan action resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], + &dev_flow->handle->dvh.push_vlan_res); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -2677,8 +2681,11 @@ struct field_modify_info modify_tcp[] = { } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next); - dev_flow->handle->dvh.push_vlan_res = cache_resource; + ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN], + &sh->push_vlan_action_list, + dev_flow->handle->dvh.push_vlan_res, + cache_resource, next); + dev_flow->dv.push_vlan_res = cache_resource; DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -3130,6 +3137,7 @@ struct field_modify_info modify_tcp[] = { { struct mlx5_flow_dv_push_vlan_action_resource res; + memset(&res, 0, sizeof(res)); res.vlan_tag = rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 | vlan->vlan_tci); @@ -7551,7 +7559,7 @@ struct field_modify_info modify_tcp[] = { (dev, attr, &vlan, dev_flow, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - handle->dvh.push_vlan_res->action; + dev_flow->dv.push_vlan_res->action; action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; break; case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: @@ -8312,6 +8320,8 @@ struct field_modify_info modify_tcp[] = { /** * Release push vlan action resource. * + * @param dev + * Pointer to Ethernet device. * @param handle * Pointer to mlx5_flow_handle. * @@ -8319,11 +8329,17 @@ struct field_modify_info modify_tcp[] = { * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_push_vlan_action_resource_release(struct mlx5_flow_handle *handle) +flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow_handle *handle) { - struct mlx5_flow_dv_push_vlan_action_resource *cache_resource = - handle->dvh.push_vlan_res; + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t idx = handle->dvh.push_vlan_res; + struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; + cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], + idx); + if (!cache_resource) + return 0; MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--", (void *)cache_resource, @@ -8331,8 +8347,10 @@ struct field_modify_info modify_tcp[] = { if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { claim_zero(mlx5_glue->destroy_flow_action (cache_resource->action)); - LIST_REMOVE(cache_resource, next); - rte_free(cache_resource); + ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], + &priv->sh->push_vlan_action_list, idx, + cache_resource, next); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); DRV_LOG(DEBUG, "push vlan action resource %p: removed", (void *)cache_resource); return 0; @@ -8412,7 +8430,8 @@ struct field_modify_info modify_tcp[] = { if (dev_handle->dvh.port_id_action) flow_dv_port_id_action_resource_release(dev_handle); if (dev_handle->dvh.push_vlan_res) - flow_dv_push_vlan_action_resource_release(dev_handle); + flow_dv_push_vlan_action_resource_release(dev, + dev_handle); if (dev_handle->dvh.tag_resource) flow_dv_tag_release(dev, dev_handle->dvh.tag_resource); From patchwork Mon Apr 13 01:11:45 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68232 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id A4237A0577; Mon, 13 Apr 2020 03:12:55 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 78BE71BEE7; Mon, 13 Apr 2020 03:12:08 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id 458241BEE1 for ; Mon, 13 Apr 2020 03:12:07 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:45 +0800 Message-Id: <1586740309-449310-7-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 06/10] net/mlx5: convert tag resource to indexed X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit convert tag resource to indexed. As tag resources are add in the hash list, to avoid introduce performance issue and keep the hash list, only the tag resource memory is allocated from indexed memory. The resources is still added to the hash list. Add four bytes index in the tag resource struct and change the tag resources in the flow handle from pointer to uint32_t seems be no benefit for tag resource, but it saves memory for flows without tag action. And also for sub flows share one tag action resource. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.c | 11 +++++++++++ drivers/net/mlx5/mlx5.h | 1 + drivers/net/mlx5/mlx5_flow.h | 7 +++++-- drivers/net/mlx5/mlx5_flow_dv.c | 27 +++++++++++++++++---------- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 3a26378..798df18 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -217,6 +217,17 @@ struct mlx5_dev_spawn_data { .free = rte_free, .type = "mlx5_push_vlan_ipool", }, + { + .size = sizeof(struct mlx5_flow_dv_tag_resource), + .trunk_size = 64, + .grow_trunk = 3, + .grow_shift = 2, + .need_lock = 0, + .release_mem_en = 1, + .malloc = rte_malloc_socket, + .free = rte_free, + .type = "mlx5_tag_ipool", + }, }; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 10e9071..6c25d6a 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -54,6 +54,7 @@ enum mlx5_mp_req_type { enum mlx5_ipool_index { MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */ MLX5_IPOOL_PUSH_VLAN, /* Pool for push vlan resource. */ + MLX5_IPOOL_TAG, /* Pool for tag resource. */ MLX5_IPOOL_MAX, }; diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 8d169fe..6364bfe 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -383,6 +383,7 @@ struct mlx5_flow_dv_tag_resource { void *action; /**< Verbs tag action object. */ rte_atomic32_t refcnt; /**< Reference counter. */ + uint32_t idx; /**< Index for the index memory pool. */ }; /* @@ -494,8 +495,8 @@ struct mlx5_flow_handle_dv { /**< Structure for VF VLAN workaround. */ uint32_t push_vlan_res; /**< Index to push VLAN action resource in cache. */ - struct mlx5_flow_dv_tag_resource *tag_resource; - /**< pointer to the tag action. */ + uint32_t tag_resource; + /**< Index to the tag action. */ }; /** Device flow handle structure: used both for creating & destroying. */ @@ -547,6 +548,8 @@ struct mlx5_flow_dv_workspace { /**< Pointer to encap/decap resource in cache. */ struct mlx5_flow_dv_push_vlan_action_resource *push_vlan_res; /**< Pointer to push VLAN action resource in cache. */ + struct mlx5_flow_dv_tag_resource *tag_resource; + /**< pointer to the tag action. */ struct mlx5_flow_dv_match_params value; /**< Holds the value that the packet is compared to. */ }; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 9965374..07ec629 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -7073,14 +7073,16 @@ struct field_modify_info modify_tcp[] = { cache_resource = container_of (entry, struct mlx5_flow_dv_tag_resource, entry); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.tag_resource = cache_resource; + dev_flow->handle->dvh.tag_resource = cache_resource->idx; + dev_flow->dv.tag_resource = cache_resource; DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); return 0; } /* Register new resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], + &dev_flow->handle->dvh.tag_resource); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -7102,7 +7104,7 @@ struct field_modify_info modify_tcp[] = { RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot insert tag"); } - dev_flow->handle->dvh.tag_resource = cache_resource; + dev_flow->dv.tag_resource = cache_resource; DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -7114,20 +7116,23 @@ struct field_modify_info modify_tcp[] = { * * @param dev * Pointer to Ethernet device. - * @param flow - * Pointer to mlx5_flow. + * @param tag_idx + * Tag index. * * @return * 1 while a reference on it exists, 0 when freed. */ static int flow_dv_tag_release(struct rte_eth_dev *dev, - struct mlx5_flow_dv_tag_resource *tag) + uint32_t tag_idx) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_dv_tag_resource *tag; - MLX5_ASSERT(tag); + tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); + if (!tag) + return 0; DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", dev->data->port_id, (void *)tag, rte_atomic32_read(&tag->refcnt)); @@ -7136,7 +7141,7 @@ struct field_modify_info modify_tcp[] = { mlx5_hlist_remove(sh->tag_table, &tag->entry); DRV_LOG(DEBUG, "port %u tag %p: removed", dev->data->port_id, (void *)tag); - rte_free(tag); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); return 0; } return 1; @@ -7433,8 +7438,9 @@ struct field_modify_info modify_tcp[] = { if (flow_dv_tag_resource_register(dev, tag_be, dev_flow, error)) return -rte_errno; + MLX5_ASSERT(dev_flow->dv.tag_resource); dev_flow->dv.actions[actions_n++] = - handle->dvh.tag_resource->action; + dev_flow->dv.tag_resource->action; break; case RTE_FLOW_ACTION_TYPE_MARK: action_flags |= MLX5_FLOW_ACTION_MARK; @@ -7460,8 +7466,9 @@ struct field_modify_info modify_tcp[] = { if (flow_dv_tag_resource_register(dev, tag_be, dev_flow, error)) return -rte_errno; + MLX5_ASSERT(dev_flow->dv.tag_resource); dev_flow->dv.actions[actions_n++] = - handle->dvh.tag_resource->action; + dev_flow->dv.tag_resource->action; break; case RTE_FLOW_ACTION_TYPE_SET_META: if (flow_dv_convert_action_set_meta From patchwork Mon Apr 13 01:11:46 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68233 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id B04C7A0577; Mon, 13 Apr 2020 03:13:07 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 5DB0A1BF59; Mon, 13 Apr 2020 03:12:11 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id 3A82B1BF26 for ; Mon, 13 Apr 2020 03:12:09 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:46 +0800 Message-Id: <1586740309-449310-8-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 07/10] net/mlx5: convert port id action to indexed X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit converts port id action to indexed. Using the uint32_t index instead of pointer saves 4 bytes memory for the flow handle. For millions flows, it will save several MBytes of memory. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.c | 11 +++++++++++ drivers/net/mlx5/mlx5.h | 4 ++-- drivers/net/mlx5/mlx5_flow.h | 6 ++++-- drivers/net/mlx5/mlx5_flow_dv.c | 43 +++++++++++++++++++++++++++++------------ 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 798df18..4477755 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -228,6 +228,17 @@ struct mlx5_dev_spawn_data { .free = rte_free, .type = "mlx5_tag_ipool", }, + { + .size = sizeof(struct mlx5_flow_dv_port_id_action_resource), + .trunk_size = 64, + .grow_trunk = 3, + .grow_shift = 2, + .need_lock = 0, + .release_mem_en = 1, + .malloc = rte_malloc_socket, + .free = rte_free, + .type = "mlx5_port_id_ipool", + }, }; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 6c25d6a..5f38587 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -55,6 +55,7 @@ enum mlx5_ipool_index { MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */ MLX5_IPOOL_PUSH_VLAN, /* Pool for push vlan resource. */ MLX5_IPOOL_TAG, /* Pool for tag resource. */ + MLX5_IPOOL_PORT_ID, /* Pool for port id resource. */ MLX5_IPOOL_MAX, }; @@ -456,8 +457,7 @@ struct mlx5_ibv_shared { uint32_t encaps_decaps; /* Encap/decap action indexed memory list. */ LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds; struct mlx5_hlist *tag_table; - LIST_HEAD(port_id_action_list, mlx5_flow_dv_port_id_action_resource) - port_id_action_list; /* List of port ID actions. */ + uint32_t port_id_action_list; /* List of port ID actions. */ uint32_t push_vlan_action_list; /* List of push VLAN actions. */ struct mlx5_flow_counter_mng cmng; /* Counters management structure. */ struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX]; diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 6364bfe..5cd9d67 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -420,7 +420,7 @@ struct mlx5_flow_dv_jump_tbl_resource { /* Port ID resource structure. */ struct mlx5_flow_dv_port_id_action_resource { - LIST_ENTRY(mlx5_flow_dv_port_id_action_resource) next; + ILIST_ENTRY(uint32_t)next; /* Pointer to next element. */ rte_atomic32_t refcnt; /**< Reference counter. */ void *action; @@ -489,7 +489,7 @@ struct mlx5_flow_handle_dv { /**< Pointer to modify header resource in cache. */ struct mlx5_flow_dv_jump_tbl_resource *jump; /**< Pointer to the jump action resource. */ - struct mlx5_flow_dv_port_id_action_resource *port_id_action; + uint32_t port_id_action; /**< Pointer to port ID action resource. */ struct mlx5_vf_vlan vf_vlan; /**< Structure for VF VLAN workaround. */ @@ -549,6 +549,8 @@ struct mlx5_flow_dv_workspace { struct mlx5_flow_dv_push_vlan_action_resource *push_vlan_res; /**< Pointer to push VLAN action resource in cache. */ struct mlx5_flow_dv_tag_resource *tag_resource; + struct mlx5_flow_dv_port_id_action_resource *port_id_action; + /**< Pointer to port ID action resource. */ /**< pointer to the tag action. */ struct mlx5_flow_dv_match_params value; /**< Holds the value that the packet is compared to. */ diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 07ec629..49c0a6d 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -2569,21 +2569,25 @@ struct field_modify_info modify_tcp[] = { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_port_id_action_resource *cache_resource; + uint32_t idx = 0; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) { + ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list, + idx, cache_resource, next) { if (resource->port_id == cache_resource->port_id) { DRV_LOG(DEBUG, "port id action resource resource %p: " "refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.port_id_action = cache_resource; + dev_flow->handle->dvh.port_id_action = idx; + dev_flow->dv.port_id_action = cache_resource; return 0; } } /* Register new port id action resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], + &dev_flow->handle->dvh.port_id_action); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -2605,8 +2609,10 @@ struct field_modify_info modify_tcp[] = { } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next); - dev_flow->handle->dvh.port_id_action = cache_resource; + ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list, + dev_flow->handle->dvh.port_id_action, cache_resource, + next); + dev_flow->dv.port_id_action = cache_resource; DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -7406,12 +7412,14 @@ struct field_modify_info modify_tcp[] = { if (flow_dv_translate_action_port_id(dev, action, &port_id, error)) return -rte_errno; + memset(&port_id_resource, 0, sizeof(port_id_resource)); port_id_resource.port_id = port_id; if (flow_dv_port_id_action_resource_register (dev, &port_id_resource, dev_flow, error)) return -rte_errno; + MLX5_ASSERT(!handle->dvh.port_id_action); dev_flow->dv.actions[actions_n++] = - handle->dvh.port_id_action->action; + dev_flow->dv.port_id_action->action; action_flags |= MLX5_FLOW_ACTION_PORT_ID; break; case RTE_FLOW_ACTION_TYPE_FLAG: @@ -8296,6 +8304,8 @@ struct field_modify_info modify_tcp[] = { /** * Release port ID action resource. * + * @param dev + * Pointer to Ethernet device. * @param handle * Pointer to mlx5_flow_handle. * @@ -8303,11 +8313,17 @@ struct field_modify_info modify_tcp[] = { * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_port_id_action_resource_release(struct mlx5_flow_handle *handle) +flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow_handle *handle) { - struct mlx5_flow_dv_port_id_action_resource *cache_resource = - handle->dvh.port_id_action; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_dv_port_id_action_resource *cache_resource; + uint32_t idx = handle->dvh.port_id_action; + cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], + idx); + if (!cache_resource) + return 0; MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", (void *)cache_resource, @@ -8315,8 +8331,10 @@ struct field_modify_info modify_tcp[] = { if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { claim_zero(mlx5_glue->destroy_flow_action (cache_resource->action)); - LIST_REMOVE(cache_resource, next); - rte_free(cache_resource); + ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID], + &priv->sh->port_id_action_list, idx, + cache_resource, next); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx); DRV_LOG(DEBUG, "port id action resource %p: removed", (void *)cache_resource); return 0; @@ -8435,7 +8453,8 @@ struct field_modify_info modify_tcp[] = { if (dev_handle->dvh.jump) flow_dv_jump_tbl_resource_release(dev, dev_handle); if (dev_handle->dvh.port_id_action) - flow_dv_port_id_action_resource_release(dev_handle); + flow_dv_port_id_action_resource_release(dev, + dev_handle); if (dev_handle->dvh.push_vlan_res) flow_dv_push_vlan_action_resource_release(dev, dev_handle); From patchwork Mon Apr 13 01:11:47 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68234 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id A4018A0577; Mon, 13 Apr 2020 03:13:16 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id AB7701BF9D; Mon, 13 Apr 2020 03:12:12 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id EA8661BF46 for ; Mon, 13 Apr 2020 03:12:10 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:47 +0800 Message-Id: <1586740309-449310-9-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 08/10] net/mlx5: convert jump resource to indexed X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit convert jump resource to indexed. The table data struct is allocated from indexed memory. As it is add in the hash list, the pointer is still used for hash list search. The index is added to the table struct, and the pointer in flow handle is decrease to uint32_t type. For flow without jump flows, it saves 4 bytes memory. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.c | 11 +++++++++++ drivers/net/mlx5/mlx5.h | 1 + drivers/net/mlx5/mlx5_flow.h | 11 +++++++---- drivers/net/mlx5/mlx5_flow_dv.c | 29 ++++++++++++++++++----------- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 4477755..9a66ed5 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -239,6 +239,17 @@ struct mlx5_dev_spawn_data { .free = rte_free, .type = "mlx5_port_id_ipool", }, + { + .size = sizeof(struct mlx5_flow_tbl_data_entry), + .trunk_size = 64, + .grow_trunk = 3, + .grow_shift = 2, + .need_lock = 0, + .release_mem_en = 1, + .malloc = rte_malloc_socket, + .free = rte_free, + .type = "mlx5_jump_ipool", + }, }; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 5f38587..2a05c09 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -56,6 +56,7 @@ enum mlx5_ipool_index { MLX5_IPOOL_PUSH_VLAN, /* Pool for push vlan resource. */ MLX5_IPOOL_TAG, /* Pool for tag resource. */ MLX5_IPOOL_PORT_ID, /* Pool for port id resource. */ + MLX5_IPOOL_JUMP, /* Pool for jump resource. */ MLX5_IPOOL_MAX, }; diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 5cd9d67..ce7e929 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -463,6 +463,7 @@ struct mlx5_flow_tbl_data_entry { /**< matchers' header associated with the flow table. */ struct mlx5_flow_dv_jump_tbl_resource jump; /**< jump resource, at most one for each table created. */ + uint32_t idx; /**< index for the indexed mempool. */ }; /* Verbs specification header. */ @@ -487,10 +488,10 @@ struct mlx5_flow_handle_dv { /**< Index to encap/decap resource in cache. */ struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; /**< Pointer to modify header resource in cache. */ - struct mlx5_flow_dv_jump_tbl_resource *jump; - /**< Pointer to the jump action resource. */ + uint32_t jump; + /**< Index to the jump action resource. */ uint32_t port_id_action; - /**< Pointer to port ID action resource. */ + /**< Index to port ID action resource. */ struct mlx5_vf_vlan vf_vlan; /**< Structure for VF VLAN workaround. */ uint32_t push_vlan_res; @@ -549,9 +550,11 @@ struct mlx5_flow_dv_workspace { struct mlx5_flow_dv_push_vlan_action_resource *push_vlan_res; /**< Pointer to push VLAN action resource in cache. */ struct mlx5_flow_dv_tag_resource *tag_resource; + /**< pointer to the tag action. */ struct mlx5_flow_dv_port_id_action_resource *port_id_action; /**< Pointer to port ID action resource. */ - /**< pointer to the tag action. */ + struct mlx5_flow_dv_jump_tbl_resource *jump; + /**< Pointer to the jump action resource. */ struct mlx5_flow_dv_match_params value; /**< Holds the value that the packet is compared to. */ }; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 49c0a6d..9fe0446 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -2540,7 +2540,8 @@ struct field_modify_info modify_tcp[] = { (void *)&tbl_data->jump, cnt); } rte_atomic32_inc(&tbl_data->jump.refcnt); - dev_flow->handle->dvh.jump = &tbl_data->jump; + dev_flow->handle->dvh.jump = tbl_data->idx; + dev_flow->dv.jump = &tbl_data->jump; return 0; } @@ -6856,6 +6857,7 @@ struct field_modify_info modify_tcp[] = { struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64); struct mlx5_flow_tbl_data_entry *tbl_data; + uint32_t idx = 0; int ret; void *domain; @@ -6866,7 +6868,7 @@ struct field_modify_info modify_tcp[] = { rte_atomic32_inc(&tbl->refcnt); return tbl; } - tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0); + tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); if (!tbl_data) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -6874,6 +6876,7 @@ struct field_modify_info modify_tcp[] = { "cannot allocate flow table data entry"); return NULL; } + tbl_data->idx = idx; tbl = &tbl_data->tbl; pos = &tbl_data->entry; if (transfer) @@ -6887,7 +6890,7 @@ struct field_modify_info modify_tcp[] = { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create flow table object"); - rte_free(tbl_data); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); return NULL; } /* @@ -6904,7 +6907,7 @@ struct field_modify_info modify_tcp[] = { RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot insert flow table data entry"); mlx5_glue->dr_destroy_flow_tbl(tbl->obj); - rte_free(tbl_data); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); } rte_atomic32_inc(&tbl->refcnt); return tbl; @@ -6939,7 +6942,8 @@ struct field_modify_info modify_tcp[] = { tbl->obj = NULL; /* remove the entry from the hash list and free memory. */ mlx5_hlist_remove(sh->flow_tbls, pos); - rte_free(tbl_data); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP], + tbl_data->idx); return 0; } return 1; @@ -7673,7 +7677,7 @@ struct field_modify_info modify_tcp[] = { "cannot create jump action."); } dev_flow->dv.actions[actions_n++] = - handle->dvh.jump->action; + dev_flow->dv.jump->action; action_flags |= MLX5_FLOW_ACTION_JUMP; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: @@ -8248,12 +8252,15 @@ struct field_modify_info modify_tcp[] = { flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { - struct mlx5_flow_dv_jump_tbl_resource *cache_resource = - handle->dvh.jump; - struct mlx5_flow_tbl_data_entry *tbl_data = - container_of(cache_resource, - struct mlx5_flow_tbl_data_entry, jump); + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_dv_jump_tbl_resource *cache_resource; + struct mlx5_flow_tbl_data_entry *tbl_data; + tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP], + handle->dvh.jump); + if (!tbl_data) + return 0; + cache_resource = &tbl_data->jump; MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", (void *)cache_resource, From patchwork Mon Apr 13 01:11:48 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Suanming Mou X-Patchwork-Id: 68235 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id B2EF7A0577; Mon, 13 Apr 2020 03:13:27 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 1C9E41BF31; Mon, 13 Apr 2020 03:12:15 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id 115AC1BFD9 for ; Mon, 13 Apr 2020 03:12:13 +0200 (CEST) From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: rasland@mellanox.com, dev@dpdk.org Date: Mon, 13 Apr 2020 09:11:48 +0800 Message-Id: <1586740309-449310-10-git-send-email-suanmingm@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> References: <1586740309-449310-1-git-send-email-suanmingm@mellanox.com> Subject: [dpdk-dev] [PATCH 09/10] net/mlx5: convert hrxq to indexed X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit converts hrxq to indexed. Using the uint32_t index instead of pointer saves 4 bytes memory for the flow handle. For millions flows, it will save several MBytes of memory. Signed-off-by: Suanming Mou Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.c | 31 +++++++++++++++++-------- drivers/net/mlx5/mlx5.h | 3 ++- drivers/net/mlx5/mlx5_flow.h | 2 +- drivers/net/mlx5/mlx5_flow_dv.c | 46 ++++++++++++++++++++----------------- drivers/net/mlx5/mlx5_flow_verbs.c | 24 +++++++++++-------- drivers/net/mlx5/mlx5_rxq.c | 47 ++++++++++++++++++++++++-------------- drivers/net/mlx5/mlx5_rxtx.h | 22 +++++++++--------- 7 files changed, 104 insertions(+), 71 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 9a66ed5..b108c0e 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -250,6 +250,17 @@ struct mlx5_dev_spawn_data { .free = rte_free, .type = "mlx5_jump_ipool", }, + { + .size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN), + .trunk_size = 64, + .grow_trunk = 3, + .grow_shift = 2, + .need_lock = 0, + .release_mem_en = 1, + .malloc = rte_malloc_socket, + .free = rte_free, + .type = "mlx5_jump_ipool", + }, }; @@ -1386,16 +1397,6 @@ struct mlx5_flow_id_pool * close(priv->nl_socket_rdma); if (priv->vmwa_context) mlx5_vlan_vmwa_exit(priv->vmwa_context); - if (priv->sh) { - /* - * Free the shared context in last turn, because the cleanup - * routines above may use some shared fields, like - * mlx5_nl_mac_addr_flush() uses ibdev_path for retrieveing - * ifindex if Netlink fails. - */ - mlx5_free_shared_ibctx(priv->sh); - priv->sh = NULL; - } ret = mlx5_hrxq_verify(dev); if (ret) DRV_LOG(WARNING, "port %u some hash Rx queue still remain", @@ -1424,6 +1425,16 @@ struct mlx5_flow_id_pool * if (ret) DRV_LOG(WARNING, "port %u some flows still remain", dev->data->port_id); + if (priv->sh) { + /* + * Free the shared context in last turn, because the cleanup + * routines above may use some shared fields, like + * mlx5_nl_mac_addr_flush() uses ibdev_path for retrieveing + * ifindex if Netlink fails. + */ + mlx5_free_shared_ibctx(priv->sh); + priv->sh = NULL; + } if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { unsigned int c = 0; uint16_t port_id; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 2a05c09..3a01c82 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -57,6 +57,7 @@ enum mlx5_ipool_index { MLX5_IPOOL_TAG, /* Pool for tag resource. */ MLX5_IPOOL_PORT_ID, /* Pool for port id resource. */ MLX5_IPOOL_JUMP, /* Pool for jump resource. */ + MLX5_IPOOL_HRXQ, /* Pool for hrxq resource. */ MLX5_IPOOL_MAX, }; @@ -534,7 +535,7 @@ struct mlx5_priv { int flow_idx; /* Intermediate device flow index. */ LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */ LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */ - LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */ + uint32_t hrxqs; /* Verbs Hash Rx queues. */ LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */ LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */ /* Indirection tables. */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index ce7e929..fcc887d 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -509,7 +509,7 @@ struct mlx5_flow_handle { uint64_t act_flags; /**< Bit-fields of detected actions, see MLX5_FLOW_ACTION_*. */ void *ib_flow; /**< Verbs flow pointer. */ - struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */ + uint32_t hrxq; /**< Hash Rx queue object index. */ struct mlx5_vf_vlan vf_vlan; /**< Structure for VF VLAN workaround. */ union { uint32_t qrss_id; /**< Uniqie Q/RSS suffix subflow tag. */ diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 9fe0446..6489213 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -8080,8 +8080,9 @@ struct field_modify_info modify_tcp[] = { if (dv->transfer) { dv->actions[n++] = priv->sh->esw_drop_action; } else { - dh->hrxq = mlx5_hrxq_drop_new(dev); - if (!dh->hrxq) { + struct mlx5_hrxq *drop_hrxq; + drop_hrxq = mlx5_hrxq_drop_new(dev); + if (!drop_hrxq) { rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -8089,28 +8090,31 @@ struct field_modify_info modify_tcp[] = { "cannot get drop hash queue"); goto error; } - dv->actions[n++] = dh->hrxq->action; + dv->actions[n++] = drop_hrxq->action; } } else if (dh->act_flags & (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) { struct mlx5_hrxq *hrxq; + uint32_t hrxq_idx; MLX5_ASSERT(flow->rss.queue); - hrxq = mlx5_hrxq_get(dev, flow->rss.key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - (*flow->rss.queue), - flow->rss.queue_num); - if (!hrxq) { - hrxq = mlx5_hrxq_new - (dev, flow->rss.key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - (*flow->rss.queue), - flow->rss.queue_num, - !!(dh->layers & - MLX5_FLOW_LAYER_TUNNEL)); + hrxq_idx = mlx5_hrxq_get(dev, flow->rss.key, + MLX5_RSS_HASH_KEY_LEN, + dev_flow->hash_fields, + (*flow->rss.queue), + flow->rss.queue_num); + if (!hrxq_idx) { + hrxq_idx = mlx5_hrxq_new + (dev, flow->rss.key, + MLX5_RSS_HASH_KEY_LEN, + dev_flow->hash_fields, + (*flow->rss.queue), + flow->rss.queue_num, + !!(dh->layers & + MLX5_FLOW_LAYER_TUNNEL)); } + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], + hrxq_idx); if (!hrxq) { rte_flow_error_set (error, rte_errno, @@ -8118,8 +8122,8 @@ struct field_modify_info modify_tcp[] = { "cannot get hash queue"); goto error; } - dh->hrxq = hrxq; - dv->actions[n++] = dh->hrxq->action; + dh->hrxq = hrxq_idx; + dv->actions[n++] = hrxq->action; } dh->ib_flow = mlx5_glue->dv_create_flow(dv_h->matcher->matcher_object, @@ -8152,7 +8156,7 @@ struct field_modify_info modify_tcp[] = { mlx5_hrxq_drop_release(dev); else mlx5_hrxq_release(dev, dh->hrxq); - dh->hrxq = NULL; + dh->hrxq = 0; } if (dh->vf_vlan.tag && dh->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); @@ -8416,7 +8420,7 @@ struct field_modify_info modify_tcp[] = { mlx5_hrxq_drop_release(dev); else mlx5_hrxq_release(dev, dh->hrxq); - dh->hrxq = NULL; + dh->hrxq = 0; } if (dh->vf_vlan.tag && dh->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index ccd3395..5f4b701 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -1656,7 +1656,7 @@ mlx5_hrxq_drop_release(dev); else mlx5_hrxq_release(dev, handle->hrxq); - handle->hrxq = NULL; + handle->hrxq = 0; } if (handle->vf_vlan.tag && handle->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); @@ -1710,6 +1710,7 @@ struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_handle *handle; struct mlx5_flow *dev_flow; + struct mlx5_hrxq *hrxq; int err; int idx; @@ -1717,8 +1718,8 @@ dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; handle = dev_flow->handle; if (handle->act_flags & MLX5_FLOW_ACTION_DROP) { - handle->hrxq = mlx5_hrxq_drop_new(dev); - if (!handle->hrxq) { + hrxq = mlx5_hrxq_drop_new(dev); + if (!hrxq) { rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -1726,22 +1727,24 @@ goto error; } } else { - struct mlx5_hrxq *hrxq; + uint32_t hrxq_idx; MLX5_ASSERT(flow->rss.queue); - hrxq = mlx5_hrxq_get(dev, flow->rss.key, + hrxq_idx = mlx5_hrxq_get(dev, flow->rss.key, MLX5_RSS_HASH_KEY_LEN, dev_flow->hash_fields, (*flow->rss.queue), flow->rss.queue_num); - if (!hrxq) - hrxq = mlx5_hrxq_new(dev, flow->rss.key, + if (!hrxq_idx) + hrxq_idx = mlx5_hrxq_new(dev, flow->rss.key, MLX5_RSS_HASH_KEY_LEN, dev_flow->hash_fields, (*flow->rss.queue), flow->rss.queue_num, !!(handle->layers & MLX5_FLOW_LAYER_TUNNEL)); + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], + hrxq_idx); if (!hrxq) { rte_flow_error_set (error, rte_errno, @@ -1749,9 +1752,10 @@ "cannot get hash queue"); goto error; } - handle->hrxq = hrxq; + handle->hrxq = hrxq_idx; } - handle->ib_flow = mlx5_glue->create_flow(handle->hrxq->qp, + MLX5_ASSERT(hrxq); + handle->ib_flow = mlx5_glue->create_flow(hrxq->qp, &dev_flow->verbs.attr); if (!handle->ib_flow) { rte_flow_error_set(error, errno, @@ -1780,7 +1784,7 @@ mlx5_hrxq_drop_release(dev); else mlx5_hrxq_release(dev, handle->hrxq); - handle->hrxq = NULL; + handle->hrxq = 0; } if (handle->vf_vlan.tag && handle->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 0a95e3c..8203025 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -2381,9 +2381,9 @@ enum mlx5_rxq_type * Tunnel type. * * @return - * The Verbs/DevX object initialised, NULL otherwise and rte_errno is set. + * The Verbs/DevX object initialised index, 0 otherwise and rte_errno is set. */ -struct mlx5_hrxq * +uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev, const uint8_t *rss_key, uint32_t rss_key_len, uint64_t hash_fields, @@ -2392,6 +2392,7 @@ struct mlx5_hrxq * { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; + uint32_t hrxq_idx = 0; struct ibv_qp *qp = NULL; struct mlx5_ind_table_obj *ind_tbl; int err; @@ -2411,7 +2412,7 @@ struct mlx5_hrxq * } if (!ind_tbl) { rte_errno = ENOMEM; - return NULL; + return 0; } if (ind_tbl->type == MLX5_IND_TBL_TYPE_IBV) { #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT @@ -2552,7 +2553,7 @@ struct mlx5_hrxq * goto error; } } - hrxq = rte_calloc(__func__, 1, sizeof(*hrxq) + rss_key_len, 0); + hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx); if (!hrxq) goto error; hrxq->ind_table = ind_tbl; @@ -2581,8 +2582,9 @@ struct mlx5_hrxq * hrxq->hash_fields = hash_fields; memcpy(hrxq->rss_key, rss_key, rss_key_len); rte_atomic32_inc(&hrxq->refcnt); - LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next); - return hrxq; + ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_HRXQ], &priv->hrxqs, hrxq_idx, + hrxq, next); + return hrxq_idx; error: err = rte_errno; /* Save rte_errno before cleanup. */ mlx5_ind_table_obj_release(dev, ind_tbl); @@ -2591,7 +2593,7 @@ struct mlx5_hrxq * else if (tir) claim_zero(mlx5_devx_cmd_destroy(tir)); rte_errno = err; /* Restore rte_errno. */ - return NULL; + return 0; } /** @@ -2608,9 +2610,9 @@ struct mlx5_hrxq * * Number of queues. * * @return - * An hash Rx queue on success. + * An hash Rx queue index on success. */ -struct mlx5_hrxq * +uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev, const uint8_t *rss_key, uint32_t rss_key_len, uint64_t hash_fields, @@ -2618,9 +2620,11 @@ struct mlx5_hrxq * { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; + uint32_t idx; queues_n = hash_fields ? queues_n : 1; - LIST_FOREACH(hrxq, &priv->hrxqs, next) { + ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_HRXQ], priv->hrxqs, idx, + hrxq, next) { struct mlx5_ind_table_obj *ind_tbl; if (hrxq->rss_key_len != rss_key_len) @@ -2637,9 +2641,9 @@ struct mlx5_hrxq * continue; } rte_atomic32_inc(&hrxq->refcnt); - return hrxq; + return idx; } - return NULL; + return 0; } /** @@ -2648,14 +2652,20 @@ struct mlx5_hrxq * * @param dev * Pointer to Ethernet device. * @param hrxq - * Pointer to Hash Rx queue to release. + * Index to Hash Rx queue to release. * * @return * 1 while a reference on it exists, 0 when freed. */ int -mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq) +mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx) { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_hrxq *hrxq; + + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); + if (!hrxq) + return 0; if (rte_atomic32_dec_and_test(&hrxq->refcnt)) { #ifdef HAVE_IBV_FLOW_DV_SUPPORT mlx5_glue->destroy_flow_action(hrxq->action); @@ -2665,8 +2675,9 @@ struct mlx5_hrxq * else /* hrxq->ind_table->type == MLX5_IND_TBL_TYPE_DEVX */ claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); mlx5_ind_table_obj_release(dev, hrxq->ind_table); - LIST_REMOVE(hrxq, next); - rte_free(hrxq); + ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_HRXQ], &priv->hrxqs, + hrxq_idx, hrxq, next); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); return 0; } claim_nonzero(mlx5_ind_table_obj_release(dev, hrxq->ind_table)); @@ -2687,9 +2698,11 @@ struct mlx5_hrxq * { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; + uint32_t idx; int ret = 0; - LIST_FOREACH(hrxq, &priv->hrxqs, next) { + ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_HRXQ], priv->hrxqs, idx, + hrxq, next) { DRV_LOG(DEBUG, "port %u hash Rx queue %p still referenced", dev->data->port_id, (void *)hrxq); diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index 939778a..b2d944b 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -231,7 +231,7 @@ struct mlx5_ind_table_obj { /* Hash Rx queue. */ struct mlx5_hrxq { - LIST_ENTRY(mlx5_hrxq) next; /* Pointer to the next element. */ + ILIST_ENTRY(uint32_t)next; /* Index to the next element. */ rte_atomic32_t refcnt; /* Reference counter. */ struct mlx5_ind_table_obj *ind_table; /* Indirection table. */ RTE_STD_C11 @@ -406,16 +406,16 @@ struct mlx5_rxq_ctrl *mlx5_rxq_hairpin_new int mlx5_rxq_verify(struct rte_eth_dev *dev); int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl); int mlx5_ind_table_obj_verify(struct rte_eth_dev *dev); -struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev, - const uint8_t *rss_key, uint32_t rss_key_len, - uint64_t hash_fields, - const uint16_t *queues, uint32_t queues_n, - int tunnel __rte_unused); -struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev, - const uint8_t *rss_key, uint32_t rss_key_len, - uint64_t hash_fields, - const uint16_t *queues, uint32_t queues_n); -int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq); +uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev, + const uint8_t *rss_key, uint32_t rss_key_len, + uint64_t hash_fields, + const uint16_t *queues, uint32_t queues_n, + int tunnel __rte_unused); +uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev, + const uint8_t *rss_key, uint32_t rss_key_len, + uint64_t hash_fields, + const uint16_t *queues, uint32_t queues_n); +int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx); int mlx5_hrxq_verify(struct rte_eth_dev *dev); enum mlx5_rxq_type mlx5_rxq_get_type(struct rte_eth_dev *dev, uint16_t idx); struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);