Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/95056/?format=api
{ "id": 95056, "url": "https://patches.dpdk.org/api/patches/95056/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20210630124609.8711-3-suanmingm@nvidia.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20210630124609.8711-3-suanmingm@nvidia.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20210630124609.8711-3-suanmingm@nvidia.com", "date": "2021-06-30T12:45:49", "name": "[v2,02/22] net/mlx5: add indexed pool local cache", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "3bd723da282c00f0fad610fb54f4ca4b4e737ef8", "submitter": { "id": 1887, "url": "https://patches.dpdk.org/api/people/1887/?format=api", "name": "Suanming Mou", "email": "suanmingm@nvidia.com" }, "delegate": { "id": 3268, "url": "https://patches.dpdk.org/api/users/3268/?format=api", "username": "rasland", "first_name": "Raslan", "last_name": "Darawsheh", "email": "rasland@nvidia.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20210630124609.8711-3-suanmingm@nvidia.com/mbox/", "series": [ { "id": 17549, "url": "https://patches.dpdk.org/api/series/17549/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=17549", "date": "2021-06-30T12:45:47", "name": "net/mlx5: insertion rate optimization", "version": 2, "mbox": "https://patches.dpdk.org/series/17549/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/95056/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/95056/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 2FE0CA0A0F;\n\tWed, 30 Jun 2021 14:46:44 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id CB90841269;\n\tWed, 30 Jun 2021 14:46:43 +0200 (CEST)", "from NAM12-DM6-obe.outbound.protection.outlook.com\n (mail-dm6nam12on2047.outbound.protection.outlook.com [40.107.243.47])\n by mails.dpdk.org (Postfix) with ESMTP id 1A8D840141\n for <dev@dpdk.org>; Wed, 30 Jun 2021 14:46:42 +0200 (CEST)", "from BN6PR20CA0072.namprd20.prod.outlook.com (2603:10b6:404:151::34)\n by DM6PR12MB3131.namprd12.prod.outlook.com (2603:10b6:5:11d::13) with\n Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.4264.18; Wed, 30 Jun\n 2021 12:46:32 +0000", "from BN8NAM11FT042.eop-nam11.prod.protection.outlook.com\n (2603:10b6:404:151:cafe::35) by BN6PR20CA0072.outlook.office365.com\n (2603:10b6:404:151::34) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.4287.23 via Frontend\n Transport; Wed, 30 Jun 2021 12:46:32 +0000", "from mail.nvidia.com (216.228.112.34) by\n BN8NAM11FT042.mail.protection.outlook.com (10.13.177.85) with Microsoft SMTP\n Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id\n 15.20.4287.22 via Frontend Transport; Wed, 30 Jun 2021 12:46:31 +0000", "from nvidia.com (172.20.187.6) by HQMAIL107.nvidia.com\n (172.20.187.13) with Microsoft SMTP Server (TLS) id 15.0.1497.2; Wed, 30 Jun\n 2021 12:46:29 +0000" ], "ARC-Seal": "i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none;\n b=MtA4rSB9kYpcNmG5bZVPRd6TIaMwFiGWble8fAmbDwDuSeOVy+lr8MDnhdxMUGmlGCCL7dVtE6pC/kq263ISDtHsPTuTJlkZUPwSprsSYSfbkk0OL2jBvibrWZS76lhundDogrcoX6tj0M4WmQgoqi1GUbiB9dYtlBEDM+ximoGtFhi4K/9t3nHZd0/v6Ynv7EtDeveIkDVjc45kb3glERlWkxjW7u43ukszQmFPGiokK7Vf1KmjvGLuWUjdg8W6ZvmvPVjOTjQzUk/fw3aJE4PSw2AtDTbQDzbdp3AjEuFGK3nSSqWpzRw5KZ/3ZrFpeQzgyYCxgdxM5dPc0Zmd4A==", "ARC-Message-Signature": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com;\n s=arcselector9901;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=G3a40XRhxQNysiw/TUmJVq9HuF3HCykU1xkDYN+Tm1A=;\n b=lWnOi4cMVh1/V3rGL95bg/E44LAXDbVglWEZTxJHp02MQwgYp+wz6kp0WcMyo9VikQSYFJ64WOGtGTJuvdXuTb7LRgVPR7jHUoslU0DLcaZrhbdXv2gLnxqcuvhIDpAqh07HHlFHq0B9mclpZgc9LuYeUaFyMTht9ERcnoMg50dVbLcaxdXggGZfp91+/d2oxUWpdRWWPiNp3JTIxKu53YHlEL7ivRHfOQAYgwi/CAjag1KIPCazoajJbKFODEDnjOCLPhZGHHLvc6EO7EOdbvK0MpI/8lBx9ZMMAHB0LnYj2tY8HadBjLc8Y2SUTvaOrZ9cVXxZo7OWRmx6PRCCeQ==", "ARC-Authentication-Results": "i=1; mx.microsoft.com 1; spf=pass (sender ip is\n 216.228.112.34) smtp.rcpttodomain=dpdk.org smtp.mailfrom=nvidia.com;\n dmarc=pass (p=none sp=none pct=100) action=none header.from=nvidia.com;\n dkim=none (message not signed); arc=none", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=Nvidia.com;\n s=selector2;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=G3a40XRhxQNysiw/TUmJVq9HuF3HCykU1xkDYN+Tm1A=;\n b=pyXRy5v21/2WdP8KBLbKD2VlkrV+WK7rz/yjbpBJvA0sNNZCF0f7Ws+JEqusNvMwxog2Dy1E/iWFuvRzySBMtXw5ZEi/BdUmvIld6qsWHDxxShcIpP7M2cg70KVsxlLLxhfR/x4MabBq3diERGeLfmQKUfZFMh48WQnegn1JiFUca6L45RWgpOI0li+qGCpas2s1JrG61BkQZcK+E1Bo14d7OdJ8O94zjel+8RPK/V/vzEaH37SVu6+ZcG5kP1jUEIWNhFus7/Qbr4m0SYY9q93xRL5MDGdCptkoj9IiYNrSTmJr/Py414NStjmW+xm55V/D+oZrQSckM+TY76PwZA==", "X-MS-Exchange-Authentication-Results": "spf=pass (sender IP is 216.228.112.34)\n smtp.mailfrom=nvidia.com; dpdk.org; dkim=none (message not signed)\n header.d=none;dpdk.org; dmarc=pass action=none header.from=nvidia.com;", "Received-SPF": "Pass (protection.outlook.com: domain of nvidia.com designates\n 216.228.112.34 as permitted sender) receiver=protection.outlook.com;\n client-ip=216.228.112.34; helo=mail.nvidia.com;", "From": "Suanming Mou <suanmingm@nvidia.com>", "To": "<viacheslavo@nvidia.com>, <matan@nvidia.com>", "CC": "<rasland@nvidia.com>, <orika@nvidia.com>, <dev@dpdk.org>", "Date": "Wed, 30 Jun 2021 15:45:49 +0300", "Message-ID": "<20210630124609.8711-3-suanmingm@nvidia.com>", "X-Mailer": "git-send-email 2.18.1", "In-Reply-To": "<20210630124609.8711-1-suanmingm@nvidia.com>", "References": "<20210527093403.1153127-1-suanmingm@nvidia.com>\n <20210630124609.8711-1-suanmingm@nvidia.com>", "MIME-Version": "1.0", "Content-Type": "text/plain", "X-Originating-IP": "[172.20.187.6]", "X-ClientProxiedBy": "HQMAIL101.nvidia.com (172.20.187.10) To\n HQMAIL107.nvidia.com (172.20.187.13)", "X-EOPAttributedMessage": "0", "X-MS-PublicTrafficType": "Email", "X-MS-Office365-Filtering-Correlation-Id": "c9ca55b6-c9c7-4c7f-16f1-08d93bc51647", "X-MS-TrafficTypeDiagnostic": "DM6PR12MB3131:", "X-Microsoft-Antispam-PRVS": "\n <DM6PR12MB31318A765728C0E8658F2F7FC1019@DM6PR12MB3131.namprd12.prod.outlook.com>", "X-MS-Oob-TLC-OOBClassifiers": "OLM:1332;", "X-MS-Exchange-SenderADCheck": "1", "X-Microsoft-Antispam": "BCL:0;", "X-Microsoft-Antispam-Message-Info": "\n +IaKjLGC0IHHS51UvbXiubwdsAoboISnJDnwk3Q+NjVxoPepZ1DjsQz4u/0LsZ8m+BByRHUFg4AHoaytgBZDGrygOU0m6cNQJQTU/8KMFhWA9n+aVbX0NPoG/oYPSm5CU7rgswBKr74IgUmK0HprMte5XKEUQw/2gtPLKYc8i837cLO2lu3urJxoeHSeHU2Y7Tro862afkPqIdhiRv73n4IPKk2jUdDQgaLjOJMjlRY8fiKjPY5fXHFP54bjSA9scPPj+IdMAJHEFg7pS7NAZIhNM0Juwq0CmmEv7nlnEdhtTWBBfoGJBrEPa02PLolqm/ApOjE8GJ6CuhrN06w+HgZ1+6jAr51IrIBlXpJZnO0vJ7TnAxc2EzkZgaUI8ahNQNfEn67c9M0l8e6U9SW82vDhDucfkiUy9AtgnCKkzNvjjpM41M0cHG732+dcOqirIdEK+k5wDRCaYJ7zWXWrffbTRyATp+kKHOAx15VBslNxCdmo+Xg6pVUTKba5N+jL6qOA3ODUofScXaz37hcPxuFo2024JmuNU1Zhn1kYbuaznfjFQxrAfB5Sd/qmnD/kDlyyqkhlnunlhTaFRjY5EtUjDUbJe51m1gKTELjBiaiCqDhIs4erzgKwLCS5Sg4BLIuE1C4tzb9mra1j7xdVLm8/iTD9paRKIWcNxMHF3cKODgmmEElp6RJiwJQFtjW0GpmhY8UbUOix9u0oRfWWSA==", "X-Forefront-Antispam-Report": "CIP:216.228.112.34; CTRY:US; LANG:en; SCL:1;\n SRV:;\n IPV:NLI; SFV:NSPM; H:mail.nvidia.com; PTR:schybrid03.nvidia.com; CAT:NONE;\n SFS:(4636009)(396003)(346002)(136003)(376002)(39860400002)(36840700001)(46966006)(336012)(54906003)(47076005)(86362001)(110136005)(6286002)(316002)(36860700001)(426003)(2616005)(5660300002)(55016002)(4326008)(30864003)(70586007)(70206006)(1076003)(36756003)(478600001)(82740400003)(2906002)(26005)(8676002)(8936002)(83380400001)(356005)(7636003)(6636002)(186003)(6666004)(82310400003)(16526019)(7696005);\n DIR:OUT; SFP:1101;", "X-OriginatorOrg": "Nvidia.com", "X-MS-Exchange-CrossTenant-OriginalArrivalTime": "30 Jun 2021 12:46:31.6350 (UTC)", "X-MS-Exchange-CrossTenant-Network-Message-Id": "\n c9ca55b6-c9c7-4c7f-16f1-08d93bc51647", "X-MS-Exchange-CrossTenant-Id": "43083d15-7273-40c1-b7db-39efd9ccc17a", "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp": "\n TenantId=43083d15-7273-40c1-b7db-39efd9ccc17a; Ip=[216.228.112.34];\n Helo=[mail.nvidia.com]", "X-MS-Exchange-CrossTenant-AuthSource": "\n BN8NAM11FT042.eop-nam11.prod.protection.outlook.com", "X-MS-Exchange-CrossTenant-AuthAs": "Anonymous", "X-MS-Exchange-CrossTenant-FromEntityHeader": "HybridOnPrem", "X-MS-Exchange-Transport-CrossTenantHeadersStamped": "DM6PR12MB3131", "Subject": "[dpdk-dev] [PATCH v2 02/22] net/mlx5: add indexed pool local cache", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "For object which wants efficient index allocate and free, local\ncache will be very helpful.\n\nTwo level cache is introduced to allocate and free the index more\nefficient. One as local and the other as global. The global cache\nis able to save all the allocated index. That means all the allocated\nindex will not be freed. Once the local cache is full, the extra\nindex will be flushed to the global cache. Once local cache is empty,\nfirst try to fetch more index from global, if global is still empty,\nallocate new trunk with more index.\n\nThis commit adds new local cache mechanism for indexed pool.\n\nSigned-off-by: Suanming Mou <suanmingm@nvidia.com>\nAcked-by: Matan Azrad <matan@nvidia.com>\n---\n drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--\n drivers/net/mlx5/mlx5_utils.h | 64 ++++++-\n 2 files changed, 372 insertions(+), 15 deletions(-)", "diff": "diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c\nindex bf2b2ebc72..215024632d 100644\n--- a/drivers/net/mlx5/mlx5_utils.c\n+++ b/drivers/net/mlx5/mlx5_utils.c\n@@ -175,14 +175,14 @@ static inline void\n mlx5_ipool_lock(struct mlx5_indexed_pool *pool)\n {\n \tif (pool->cfg.need_lock)\n-\t\trte_spinlock_lock(&pool->lock);\n+\t\trte_spinlock_lock(&pool->rsz_lock);\n }\n \n static inline void\n mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)\n {\n \tif (pool->cfg.need_lock)\n-\t\trte_spinlock_unlock(&pool->lock);\n+\t\trte_spinlock_unlock(&pool->rsz_lock);\n }\n \n static inline uint32_t\n@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)\n \tuint32_t i;\n \n \tif (!cfg || (!cfg->malloc ^ !cfg->free) ||\n+\t (cfg->per_core_cache && cfg->release_mem_en) ||\n \t (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||\n \t ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))\n \t\treturn NULL;\n@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)\n \t\tpool->cfg.malloc = mlx5_malloc;\n \t\tpool->cfg.free = mlx5_free;\n \t}\n-\tpool->free_list = TRUNK_INVALID;\n \tif (pool->cfg.need_lock)\n-\t\trte_spinlock_init(&pool->lock);\n+\t\trte_spinlock_init(&pool->rsz_lock);\n \t/*\n \t * Initialize the dynamic grow trunk size lookup table to have a quick\n \t * lookup for the trunk entry index offset.\n@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)\n \tif (!pool->cfg.max_idx)\n \t\tpool->cfg.max_idx =\n \t\t\tmlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);\n+\tif (!cfg->per_core_cache)\n+\t\tpool->free_list = TRUNK_INVALID;\n \treturn pool;\n }\n \n@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)\n \treturn 0;\n }\n \n+static inline struct mlx5_indexed_cache *\n+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)\n+{\n+\tstruct mlx5_indexed_cache *gc, *lc, *olc = NULL;\n+\n+\tlc = pool->cache[cidx]->lc;\n+\tgc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);\n+\tif (gc && lc != gc) {\n+\t\tmlx5_ipool_lock(pool);\n+\t\tif (lc && !(--lc->ref_cnt))\n+\t\t\tolc = lc;\n+\t\tlc = pool->gc;\n+\t\tlc->ref_cnt++;\n+\t\tpool->cache[cidx]->lc = lc;\n+\t\tmlx5_ipool_unlock(pool);\n+\t\tif (olc)\n+\t\t\tpool->cfg.free(olc);\n+\t}\n+\treturn lc;\n+}\n+\n+static uint32_t\n+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)\n+{\n+\tstruct mlx5_indexed_trunk *trunk;\n+\tstruct mlx5_indexed_cache *p, *lc, *olc = NULL;\n+\tsize_t trunk_size = 0;\n+\tsize_t data_size;\n+\tuint32_t cur_max_idx, trunk_idx, trunk_n;\n+\tuint32_t fetch_size, ts_idx, i;\n+\tint n_grow;\n+\n+check_again:\n+\tp = NULL;\n+\tfetch_size = 0;\n+\t/*\n+\t * Fetch new index from global if possible. First round local\n+\t * cache will be NULL.\n+\t */\n+\tlc = pool->cache[cidx]->lc;\n+\tmlx5_ipool_lock(pool);\n+\t/* Try to update local cache first. */\n+\tif (likely(pool->gc)) {\n+\t\tif (lc != pool->gc) {\n+\t\t\tif (lc && !(--lc->ref_cnt))\n+\t\t\t\tolc = lc;\n+\t\t\tlc = pool->gc;\n+\t\t\tlc->ref_cnt++;\n+\t\t\tpool->cache[cidx]->lc = lc;\n+\t\t}\n+\t\tif (lc->len) {\n+\t\t\t/* Use the updated local cache to fetch index. */\n+\t\t\tfetch_size = pool->cfg.per_core_cache >> 2;\n+\t\t\tif (lc->len < fetch_size)\n+\t\t\t\tfetch_size = lc->len;\n+\t\t\tlc->len -= fetch_size;\n+\t\t\tmemcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],\n+\t\t\t sizeof(uint32_t) * fetch_size);\n+\t\t}\n+\t}\n+\tmlx5_ipool_unlock(pool);\n+\tif (unlikely(olc)) {\n+\t\tpool->cfg.free(olc);\n+\t\tolc = NULL;\n+\t}\n+\tif (fetch_size) {\n+\t\tpool->cache[cidx]->len = fetch_size - 1;\n+\t\treturn pool->cache[cidx]->idx[pool->cache[cidx]->len];\n+\t}\n+\ttrunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,\n+\t\t\t __ATOMIC_ACQUIRE) : 0;\n+\ttrunk_n = lc ? lc->n_trunk : 0;\n+\tcur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);\n+\t/* Check if index reach maximum. */\n+\tif (trunk_idx == TRUNK_MAX_IDX ||\n+\t cur_max_idx >= pool->cfg.max_idx)\n+\t\treturn 0;\n+\t/* No enough space in trunk array, resize the trunks array. */\n+\tif (trunk_idx == trunk_n) {\n+\t\tn_grow = trunk_idx ? trunk_idx :\n+\t\t\t RTE_CACHE_LINE_SIZE / sizeof(void *);\n+\t\tcur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);\n+\t\t/* Resize the trunk array. */\n+\t\tp = pool->cfg.malloc(0, ((trunk_idx + n_grow) *\n+\t\t\tsizeof(struct mlx5_indexed_trunk *)) +\n+\t\t\t(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),\n+\t\t\tRTE_CACHE_LINE_SIZE, rte_socket_id());\n+\t\tif (!p)\n+\t\t\treturn 0;\n+\t\tp->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];\n+\t\tif (lc)\n+\t\t\tmemcpy(p->trunks, lc->trunks, trunk_idx *\n+\t\t sizeof(struct mlx5_indexed_trunk *));\n+#ifdef RTE_LIBRTE_MLX5_DEBUG\n+\t\tmemset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,\n+\t\t\tn_grow * sizeof(void *));\n+#endif\n+\t\tp->n_trunk_valid = trunk_idx;\n+\t\tp->n_trunk = trunk_n + n_grow;\n+\t\tp->len = 0;\n+\t}\n+\t/* Prepare the new trunk. */\n+\ttrunk_size = sizeof(*trunk);\n+\tdata_size = mlx5_trunk_size_get(pool, trunk_idx);\n+\ttrunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);\n+\ttrunk = pool->cfg.malloc(0, trunk_size,\n+\t\t\t\t RTE_CACHE_LINE_SIZE, rte_socket_id());\n+\tif (unlikely(!trunk)) {\n+\t\tpool->cfg.free(p);\n+\t\treturn 0;\n+\t}\n+\ttrunk->idx = trunk_idx;\n+\ttrunk->free = data_size;\n+\tmlx5_ipool_lock(pool);\n+\t/*\n+\t * Double check if trunks has been updated or have available index.\n+\t * During the new trunk allocate, index may still be flushed to the\n+\t * global cache. So also need to check the pool->gc->len.\n+\t */\n+\tif (pool->gc && (lc != pool->gc ||\n+\t lc->n_trunk_valid != trunk_idx ||\n+\t pool->gc->len)) {\n+\t\tmlx5_ipool_unlock(pool);\n+\t\tif (p)\n+\t\t\tpool->cfg.free(p);\n+\t\tpool->cfg.free(trunk);\n+\t\tgoto check_again;\n+\t}\n+\t/* Resize the trunk array and update local cache first. */\n+\tif (p) {\n+\t\tif (lc && !(--lc->ref_cnt))\n+\t\t\tolc = lc;\n+\t\tlc = p;\n+\t\tlc->ref_cnt = 1;\n+\t\tpool->cache[cidx]->lc = lc;\n+\t\t__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);\n+\t}\n+\t/* Add trunk to trunks array. */\n+\tlc->trunks[trunk_idx] = trunk;\n+\t__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);\n+\t/* Enqueue half of the index to global. */\n+\tts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;\n+\tfetch_size = trunk->free >> 1;\n+\tfor (i = 0; i < fetch_size; i++)\n+\t\tlc->idx[i] = ts_idx + i;\n+\tlc->len = fetch_size;\n+\tmlx5_ipool_unlock(pool);\n+\t/* Copy left half - 1 to local cache index array. */\n+\tpool->cache[cidx]->len = trunk->free - fetch_size - 1;\n+\tts_idx += fetch_size;\n+\tfor (i = 0; i < pool->cache[cidx]->len; i++)\n+\t\tpool->cache[cidx]->idx[i] = ts_idx + i;\n+\tif (olc)\n+\t\tpool->cfg.free(olc);\n+\treturn ts_idx + i;\n+}\n+\n+static void *\n+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)\n+{\n+\tstruct mlx5_indexed_trunk *trunk;\n+\tstruct mlx5_indexed_cache *lc;\n+\tuint32_t trunk_idx;\n+\tuint32_t entry_idx;\n+\tint cidx;\n+\n+\tMLX5_ASSERT(idx);\n+\tcidx = rte_lcore_index(rte_lcore_id());\n+\tif (unlikely(cidx == -1)) {\n+\t\trte_errno = ENOTSUP;\n+\t\treturn NULL;\n+\t}\n+\tlc = mlx5_ipool_update_global_cache(pool, cidx);\n+\tidx -= 1;\n+\ttrunk_idx = mlx5_trunk_idx_get(pool, idx);\n+\ttrunk = lc->trunks[trunk_idx];\n+\tMLX5_ASSERT(trunk);\n+\tentry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);\n+\treturn &trunk->data[entry_idx * pool->cfg.size];\n+}\n+\n+static void *\n+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)\n+{\n+\tint cidx;\n+\n+\tcidx = rte_lcore_index(rte_lcore_id());\n+\tif (unlikely(cidx == -1)) {\n+\t\trte_errno = ENOTSUP;\n+\t\treturn NULL;\n+\t}\n+\tif (unlikely(!pool->cache[cidx])) {\n+\t\tpool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,\n+\t\t\tsizeof(struct mlx5_ipool_per_lcore) +\n+\t\t\t(pool->cfg.per_core_cache * sizeof(uint32_t)),\n+\t\t\tRTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);\n+\t\tif (!pool->cache[cidx]) {\n+\t\t\tDRV_LOG(ERR, \"Ipool cache%d allocate failed\\n\", cidx);\n+\t\t\treturn NULL;\n+\t\t}\n+\t} else if (pool->cache[cidx]->len) {\n+\t\tpool->cache[cidx]->len--;\n+\t\t*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];\n+\t\treturn mlx5_ipool_get_cache(pool, *idx);\n+\t}\n+\t/* Not enough idx in global cache. Keep fetching from global. */\n+\t*idx = mlx5_ipool_allocate_from_global(pool, cidx);\n+\tif (unlikely(!(*idx)))\n+\t\treturn NULL;\n+\treturn mlx5_ipool_get_cache(pool, *idx);\n+}\n+\n+static void\n+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)\n+{\n+\tint cidx;\n+\tstruct mlx5_ipool_per_lcore *ilc;\n+\tstruct mlx5_indexed_cache *gc, *olc = NULL;\n+\tuint32_t reclaim_num = 0;\n+\n+\tMLX5_ASSERT(idx);\n+\tcidx = rte_lcore_index(rte_lcore_id());\n+\tif (unlikely(cidx == -1)) {\n+\t\trte_errno = ENOTSUP;\n+\t\treturn;\n+\t}\n+\t/*\n+\t * When index was allocated on core A but freed on core B. In this\n+\t * case check if local cache on core B was allocated before.\n+\t */\n+\tif (unlikely(!pool->cache[cidx])) {\n+\t\tpool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,\n+\t\t\tsizeof(struct mlx5_ipool_per_lcore) +\n+\t\t\t(pool->cfg.per_core_cache * sizeof(uint32_t)),\n+\t\t\tRTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);\n+\t\tif (!pool->cache[cidx]) {\n+\t\t\tDRV_LOG(ERR, \"Ipool cache%d allocate failed\\n\", cidx);\n+\t\t\treturn;\n+\t\t}\n+\t}\n+\t/* Try to enqueue to local index cache. */\n+\tif (pool->cache[cidx]->len < pool->cfg.per_core_cache) {\n+\t\tpool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;\n+\t\tpool->cache[cidx]->len++;\n+\t\treturn;\n+\t}\n+\tilc = pool->cache[cidx];\n+\treclaim_num = pool->cfg.per_core_cache >> 2;\n+\tilc->len -= reclaim_num;\n+\t/* Local index cache full, try with global index cache. */\n+\tmlx5_ipool_lock(pool);\n+\tgc = pool->gc;\n+\tif (ilc->lc != gc) {\n+\t\tif (!(--ilc->lc->ref_cnt))\n+\t\t\tolc = ilc->lc;\n+\t\tgc->ref_cnt++;\n+\t\tilc->lc = gc;\n+\t}\n+\tmemcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],\n+\t reclaim_num * sizeof(uint32_t));\n+\tgc->len += reclaim_num;\n+\tmlx5_ipool_unlock(pool);\n+\tif (olc)\n+\t\tpool->cfg.free(olc);\n+\tpool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;\n+\tpool->cache[cidx]->len++;\n+}\n+\n void *\n mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)\n {\n@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)\n \tuint32_t iidx = 0;\n \tvoid *p;\n \n+\tif (pool->cfg.per_core_cache)\n+\t\treturn mlx5_ipool_malloc_cache(pool, idx);\n \tmlx5_ipool_lock(pool);\n \tif (pool->free_list == TRUNK_INVALID) {\n \t\t/* If no available trunks, grow new. */\n@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)\n \n \tif (!idx)\n \t\treturn;\n+\tif (pool->cfg.per_core_cache) {\n+\t\tmlx5_ipool_free_cache(pool, idx);\n+\t\treturn;\n+\t}\n \tidx -= 1;\n \tmlx5_ipool_lock(pool);\n \ttrunk_idx = mlx5_trunk_idx_get(pool, idx);\n@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)\n \n \tif (!idx)\n \t\treturn NULL;\n+\tif (pool->cfg.per_core_cache)\n+\t\treturn mlx5_ipool_get_cache(pool, idx);\n \tidx -= 1;\n \tmlx5_ipool_lock(pool);\n \ttrunk_idx = mlx5_trunk_idx_get(pool, idx);\n@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)\n int\n mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)\n {\n-\tstruct mlx5_indexed_trunk **trunks;\n-\tuint32_t i;\n+\tstruct mlx5_indexed_trunk **trunks = NULL;\n+\tstruct mlx5_indexed_cache *gc = pool->gc;\n+\tuint32_t i, n_trunk_valid = 0;\n \n \tMLX5_ASSERT(pool);\n \tmlx5_ipool_lock(pool);\n-\ttrunks = pool->trunks;\n-\tfor (i = 0; i < pool->n_trunk; i++) {\n+\tif (pool->cfg.per_core_cache) {\n+\t\tfor (i = 0; i < RTE_MAX_LCORE; i++) {\n+\t\t\t/*\n+\t\t\t * Free only old global cache. Pool gc will be\n+\t\t\t * freed at last.\n+\t\t\t */\n+\t\t\tif (pool->cache[i]) {\n+\t\t\t\tif (pool->cache[i]->lc &&\n+\t\t\t\t pool->cache[i]->lc != pool->gc &&\n+\t\t\t\t (!(--pool->cache[i]->lc->ref_cnt)))\n+\t\t\t\t\tpool->cfg.free(pool->cache[i]->lc);\n+\t\t\t\tpool->cfg.free(pool->cache[i]);\n+\t\t\t}\n+\t\t}\n+\t\tif (gc) {\n+\t\t\ttrunks = gc->trunks;\n+\t\t\tn_trunk_valid = gc->n_trunk_valid;\n+\t\t}\n+\t} else {\n+\t\tgc = NULL;\n+\t\ttrunks = pool->trunks;\n+\t\tn_trunk_valid = pool->n_trunk_valid;\n+\t}\n+\tfor (i = 0; i < n_trunk_valid; i++) {\n \t\tif (trunks[i])\n \t\t\tpool->cfg.free(trunks[i]);\n \t}\n-\tif (!pool->trunks)\n-\t\tpool->cfg.free(pool->trunks);\n+\tif (!gc && trunks)\n+\t\tpool->cfg.free(trunks);\n+\tif (gc)\n+\t\tpool->cfg.free(gc);\n \tmlx5_ipool_unlock(pool);\n \tmlx5_free(pool);\n \treturn 0;\ndiff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h\nindex 15870e14c2..0469062695 100644\n--- a/drivers/net/mlx5/mlx5_utils.h\n+++ b/drivers/net/mlx5/mlx5_utils.h\n@@ -209,6 +209,11 @@ struct mlx5_indexed_pool_config {\n \t/* Lock is needed for multiple thread usage. */\n \tuint32_t release_mem_en:1; /* Rlease trunk when it is free. */\n \tuint32_t max_idx; /* The maximum index can be allocated. */\n+\tuint32_t per_core_cache;\n+\t/*\n+\t * Cache entry number per core for performance. Should not be\n+\t * set with release_mem_en.\n+\t */\n \tconst char *type; /* Memory allocate type name. */\n \tvoid *(*malloc)(uint32_t flags, size_t size, unsigned int align,\n \t\t\tint socket);\n@@ -225,14 +230,39 @@ struct mlx5_indexed_trunk {\n \tuint8_t data[] __rte_cache_aligned; /* Entry data start. */\n };\n \n+struct mlx5_indexed_cache {\n+\tstruct mlx5_indexed_trunk **trunks;\n+\tvolatile uint32_t n_trunk_valid; /* Trunks allocated. */\n+\tuint32_t n_trunk; /* Trunk pointer array size. */\n+\tuint32_t ref_cnt;\n+\tuint32_t len;\n+\tuint32_t idx[];\n+};\n+\n+struct mlx5_ipool_per_lcore {\n+\tstruct mlx5_indexed_cache *lc;\n+\tuint32_t len; /**< Current cache count. */\n+\tuint32_t idx[]; /**< Cache objects. */\n+};\n+\n struct mlx5_indexed_pool {\n \tstruct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */\n-\trte_spinlock_t lock; /* Pool lock for multiple thread usage. */\n-\tuint32_t n_trunk_valid; /* Trunks allocated. */\n-\tuint32_t n_trunk; /* Trunk pointer array size. */\n+\trte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */\n \t/* Dim of trunk pointer array. */\n-\tstruct mlx5_indexed_trunk **trunks;\n-\tuint32_t free_list; /* Index to first free trunk. */\n+\tunion {\n+\t\tstruct {\n+\t\t\tuint32_t n_trunk_valid; /* Trunks allocated. */\n+\t\t\tuint32_t n_trunk; /* Trunk pointer array size. */\n+\t\t\tstruct mlx5_indexed_trunk **trunks;\n+\t\t\tuint32_t free_list; /* Index to first free trunk. */\n+\t\t};\n+\t\tstruct {\n+\t\t\tstruct mlx5_indexed_cache *gc;\n+\t\t\t/* Global cache. */\n+\t\t\tstruct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];\n+\t\t\t/* Local cache. */\n+\t\t};\n+\t};\n #ifdef POOL_DEBUG\n \tuint32_t n_entry;\n \tuint32_t trunk_new;\n@@ -542,6 +572,30 @@ int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool);\n */\n void mlx5_ipool_dump(struct mlx5_indexed_pool *pool);\n \n+/**\n+ * This function flushes all the cache index back to pool trunk.\n+ *\n+ * @param pool\n+ * Pointer to the index memory pool handler.\n+ *\n+ */\n+\n+void mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool);\n+\n+/**\n+ * This function gets the available entry from pos.\n+ *\n+ * @param pool\n+ * Pointer to the index memory pool handler.\n+ * @param pos\n+ * Pointer to the index position start from.\n+ *\n+ * @return\n+ * - Pointer to the next available entry.\n+ *\n+ */\n+void *mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos);\n+\n /**\n * This function allocates new empty Three-level table.\n *\n", "prefixes": [ "v2", "02/22" ] }