Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/76968/?format=api
{ "id": 76968, "url": "http://patches.dpdk.org/api/patches/76968/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20200908201830.74206-36-cristian.dumitrescu@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20200908201830.74206-36-cristian.dumitrescu@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20200908201830.74206-36-cristian.dumitrescu@intel.com", "date": "2020-09-08T20:18:24", "name": "[v3,35/41] table: add exact match SWX table", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "3abcd3957c5c29c83e94bafb533fe628ac3abd3f", "submitter": { "id": 19, "url": "http://patches.dpdk.org/api/people/19/?format=api", "name": "Cristian Dumitrescu", "email": "cristian.dumitrescu@intel.com" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20200908201830.74206-36-cristian.dumitrescu@intel.com/mbox/", "series": [ { "id": 12034, "url": "http://patches.dpdk.org/api/series/12034/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=12034", "date": "2020-09-08T20:17:52", "name": "Pipeline alignment with the P4 language", "version": 3, "mbox": "http://patches.dpdk.org/series/12034/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/76968/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/76968/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id AFF50A04B1;\n\tTue, 8 Sep 2020 22:24:40 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 8962A1C299;\n\tTue, 8 Sep 2020 22:19:45 +0200 (CEST)", "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n by dpdk.org (Postfix) with ESMTP id C91981C12A\n for <dev@dpdk.org>; Tue, 8 Sep 2020 22:19:07 +0200 (CEST)", "from fmsmga006.fm.intel.com ([10.253.24.20])\n by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 08 Sep 2020 13:19:07 -0700", "from silpixa00400573.ir.intel.com (HELO\n silpixa00400573.ger.corp.intel.com) ([10.237.223.107])\n by fmsmga006.fm.intel.com with ESMTP; 08 Sep 2020 13:19:06 -0700" ], "IronPort-SDR": [ "\n 7RUa+yA612EmRpoS0M83caHIh3bnCiQ+bfZBbWF6PjyDkFGFfJlBdxzSxFD/5utK9lVJMJeFya\n zhQlAz2Hpq6Q==", "\n V/Lx5nlKOF8hHO5F0j1h+iiz2zjsxSFTAXq9neabF7lR6N5/QTBybj2C4Y17z9uU4BR62EPTc3\n xh5xE3iuJgiA==" ], "X-IronPort-AV": [ "E=McAfee;i=\"6000,8403,9738\"; a=\"145939453\"", "E=Sophos;i=\"5.76,407,1592895600\"; d=\"scan'208\";a=\"145939453\"", "E=Sophos;i=\"5.76,406,1592895600\"; d=\"scan'208\";a=\"504493549\"" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "From": "Cristian Dumitrescu <cristian.dumitrescu@intel.com>", "To": "dev@dpdk.org", "Date": "Tue, 8 Sep 2020 21:18:24 +0100", "Message-Id": "<20200908201830.74206-36-cristian.dumitrescu@intel.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20200908201830.74206-1-cristian.dumitrescu@intel.com>", "References": "<20200907214032.95052-2-cristian.dumitrescu@intel.com>\n <20200908201830.74206-1-cristian.dumitrescu@intel.com>", "Subject": "[dpdk-dev] [PATCH v3 35/41] table: add exact match SWX table", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add the exact match table type for the SWX pipeline. Used under the\nhood by the SWX pipeline table instruction.\n\nSigned-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com>\n---\n lib/librte_table/meson.build | 6 +-\n lib/librte_table/rte_swx_table_em.c | 851 +++++++++++++++++++++++++\n lib/librte_table/rte_swx_table_em.h | 30 +\n lib/librte_table/rte_table_version.map | 7 +\n 4 files changed, 892 insertions(+), 2 deletions(-)\n create mode 100644 lib/librte_table/rte_swx_table_em.c\n create mode 100644 lib/librte_table/rte_swx_table_em.h", "diff": "diff --git a/lib/librte_table/meson.build b/lib/librte_table/meson.build\nindex b9d4fe3dc..d69678386 100644\n--- a/lib/librte_table/meson.build\n+++ b/lib/librte_table/meson.build\n@@ -11,7 +11,8 @@ sources = files('rte_table_acl.c',\n \t\t'rte_table_hash_ext.c',\n \t\t'rte_table_hash_lru.c',\n \t\t'rte_table_array.c',\n-\t\t'rte_table_stub.c')\n+\t\t'rte_table_stub.c',\n+\t\t'rte_swx_table_em.c',)\n headers = files('rte_table.h',\n \t\t'rte_table_acl.h',\n \t\t'rte_table_lpm.h',\n@@ -23,7 +24,8 @@ headers = files('rte_table.h',\n \t\t'rte_lru.h',\n \t\t'rte_table_array.h',\n \t\t'rte_table_stub.h',\n-\t\t'rte_swx_table.h',)\n+\t\t'rte_swx_table.h',\n+\t\t'rte_swx_table_em.h',)\n deps += ['mbuf', 'port', 'lpm', 'hash', 'acl']\n \n if arch_subdir == 'x86'\ndiff --git a/lib/librte_table/rte_swx_table_em.c b/lib/librte_table/rte_swx_table_em.c\nnew file mode 100644\nindex 000000000..85c77ad03\n--- /dev/null\n+++ b/lib/librte_table/rte_swx_table_em.c\n@@ -0,0 +1,851 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(c) 2020 Intel Corporation\n+ */\n+#include <stdlib.h>\n+#include <string.h>\n+#include <stdio.h>\n+#include <errno.h>\n+\n+#include <rte_common.h>\n+#include <rte_prefetch.h>\n+\n+#include \"rte_swx_table_em.h\"\n+\n+#define CHECK(condition, err_code) \\\n+do { \\\n+\tif (!(condition)) \\\n+\t\treturn -(err_code); \\\n+} while (0)\n+\n+#ifndef RTE_SWX_TABLE_EM_USE_HUGE_PAGES\n+#define RTE_SWX_TABLE_EM_USE_HUGE_PAGES 1\n+#endif\n+\n+#if RTE_SWX_TABLE_EM_USE_HUGE_PAGES\n+\n+#include <rte_malloc.h>\n+\n+static void *\n+env_malloc(size_t size, size_t alignment, int numa_node)\n+{\n+\treturn rte_zmalloc_socket(NULL, size, alignment, numa_node);\n+}\n+\n+static void\n+env_free(void *start, size_t size __rte_unused)\n+{\n+\trte_free(start);\n+}\n+\n+#else\n+\n+#include <numa.h>\n+\n+static void *\n+env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)\n+{\n+\treturn numa_alloc_onnode(size, numa_node);\n+}\n+\n+static void\n+env_free(void *start, size_t size)\n+{\n+\tnuma_free(start, size);\n+}\n+\n+#endif\n+\n+#if defined(RTE_ARCH_X86_64)\n+\n+#include <x86intrin.h>\n+\n+#define crc32_u64(crc, v) _mm_crc32_u64(crc, v)\n+\n+#else\n+\n+static inline uint64_t\n+crc32_u64_generic(uint64_t crc, uint64_t value)\n+{\n+\tint i;\n+\n+\tcrc = (crc & 0xFFFFFFFFLLU) ^ value;\n+\tfor (i = 63; i >= 0; i--) {\n+\t\tuint64_t mask;\n+\n+\t\tmask = -(crc & 1LLU);\n+\t\tcrc = (crc >> 1LLU) ^ (0x82F63B78LLU & mask);\n+\t}\n+\n+\treturn crc;\n+}\n+\n+#define crc32_u64(crc, v) crc32_u64_generic(crc, v)\n+\n+#endif\n+\n+/* Key size needs to be one of: 8, 16, 32 or 64. */\n+static inline uint32_t\n+hash(void *key, void *key_mask, uint32_t key_size, uint32_t seed)\n+{\n+\tuint64_t *k = key;\n+\tuint64_t *m = key_mask;\n+\tuint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;\n+\n+\tswitch (key_size) {\n+\tcase 8:\n+\t\tcrc0 = crc32_u64(seed, k[0] & m[0]);\n+\t\treturn crc0;\n+\n+\tcase 16:\n+\t\tk0 = k[0] & m[0];\n+\n+\t\tcrc0 = crc32_u64(k0, seed);\n+\t\tcrc1 = crc32_u64(k0 >> 32, k[1] & m[1]);\n+\n+\t\tcrc0 ^= crc1;\n+\n+\t\treturn crc0;\n+\n+\tcase 32:\n+\t\tk0 = k[0] & m[0];\n+\t\tk2 = k[2] & m[2];\n+\n+\t\tcrc0 = crc32_u64(k0, seed);\n+\t\tcrc1 = crc32_u64(k0 >> 32, k[1] & m[1]);\n+\n+\t\tcrc2 = crc32_u64(k2, k[3] & m[3]);\n+\t\tcrc3 = k2 >> 32;\n+\n+\t\tcrc0 = crc32_u64(crc0, crc1);\n+\t\tcrc1 = crc32_u64(crc2, crc3);\n+\n+\t\tcrc0 ^= crc1;\n+\n+\t\treturn crc0;\n+\n+\tcase 64:\n+\t\tk0 = k[0] & m[0];\n+\t\tk2 = k[2] & m[2];\n+\t\tk5 = k[5] & m[5];\n+\n+\t\tcrc0 = crc32_u64(k0, seed);\n+\t\tcrc1 = crc32_u64(k0 >> 32, k[1] & m[1]);\n+\n+\t\tcrc2 = crc32_u64(k2, k[3] & m[3]);\n+\t\tcrc3 = crc32_u64(k2 >> 32, k[4] & m[4]);\n+\n+\t\tcrc4 = crc32_u64(k5, k[6] & m[6]);\n+\t\tcrc5 = crc32_u64(k5 >> 32, k[7] & m[7]);\n+\n+\t\tcrc0 = crc32_u64(crc0, (crc1 << 32) ^ crc2);\n+\t\tcrc1 = crc32_u64(crc3, (crc4 << 32) ^ crc5);\n+\n+\t\tcrc0 ^= crc1;\n+\n+\t\treturn crc0;\n+\n+\tdefault:\n+\t\tcrc0 = 0;\n+\t\treturn crc0;\n+\t}\n+}\n+\n+/* n_bytes needs to be a multiple of 8 bytes. */\n+static void\n+keycpy(void *dst, void *src, void *src_mask, uint32_t n_bytes)\n+{\n+\tuint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask;\n+\tuint32_t i;\n+\n+\tfor (i = 0; i < n_bytes / sizeof(uint64_t); i++)\n+\t\tdst64[i] = src64[i] & src_mask64[i];\n+}\n+\n+/*\n+ * Return: 0 = Keys are NOT equal; 1 = Keys are equal.\n+ */\n+static inline uint32_t\n+keycmp(void *a, void *b, void *b_mask, uint32_t n_bytes)\n+{\n+\tuint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask;\n+\n+\tswitch (n_bytes) {\n+\tcase 8: {\n+\t\tuint64_t xor0 = a64[0] ^ (b64[0] & b_mask64[0]);\n+\t\tuint32_t result = 1;\n+\n+\t\tif (xor0)\n+\t\t\tresult = 0;\n+\t\treturn result;\n+\t}\n+\n+\tcase 16: {\n+\t\tuint64_t xor0 = a64[0] ^ (b64[0] & b_mask64[0]);\n+\t\tuint64_t xor1 = a64[1] ^ (b64[1] & b_mask64[1]);\n+\t\tuint64_t or = xor0 | xor1;\n+\t\tuint32_t result = 1;\n+\n+\t\tif (or)\n+\t\t\tresult = 0;\n+\t\treturn result;\n+\t}\n+\n+\tcase 32: {\n+\t\tuint64_t xor0 = a64[0] ^ (b64[0] & b_mask64[0]);\n+\t\tuint64_t xor1 = a64[1] ^ (b64[1] & b_mask64[1]);\n+\t\tuint64_t xor2 = a64[2] ^ (b64[2] & b_mask64[2]);\n+\t\tuint64_t xor3 = a64[3] ^ (b64[3] & b_mask64[3]);\n+\t\tuint64_t or = (xor0 | xor1) | (xor2 | xor3);\n+\t\tuint32_t result = 1;\n+\n+\t\tif (or)\n+\t\t\tresult = 0;\n+\t\treturn result;\n+\t}\n+\n+\tcase 64: {\n+\t\tuint64_t xor0 = a64[0] ^ (b64[0] & b_mask64[0]);\n+\t\tuint64_t xor1 = a64[1] ^ (b64[1] & b_mask64[1]);\n+\t\tuint64_t xor2 = a64[2] ^ (b64[2] & b_mask64[2]);\n+\t\tuint64_t xor3 = a64[3] ^ (b64[3] & b_mask64[3]);\n+\t\tuint64_t xor4 = a64[4] ^ (b64[4] & b_mask64[4]);\n+\t\tuint64_t xor5 = a64[5] ^ (b64[5] & b_mask64[5]);\n+\t\tuint64_t xor6 = a64[6] ^ (b64[6] & b_mask64[6]);\n+\t\tuint64_t xor7 = a64[7] ^ (b64[7] & b_mask64[7]);\n+\t\tuint64_t or = ((xor0 | xor1) | (xor2 | xor3)) |\n+\t\t\t ((xor4 | xor5) | (xor6 | xor7));\n+\t\tuint32_t result = 1;\n+\n+\t\tif (or)\n+\t\t\tresult = 0;\n+\t\treturn result;\n+\t}\n+\n+\tdefault: {\n+\t\tuint32_t i;\n+\n+\t\tfor (i = 0; i < n_bytes / sizeof(uint64_t); i++)\n+\t\t\tif (a64[i] != (b64[i] & b_mask64[i]))\n+\t\t\t\treturn 0;\n+\t\treturn 1;\n+\t}\n+\t}\n+}\n+\n+#define KEYS_PER_BUCKET 4\n+\n+struct bucket_extension {\n+\tstruct bucket_extension *next;\n+\tuint16_t sig[KEYS_PER_BUCKET];\n+\tuint32_t key_id[KEYS_PER_BUCKET];\n+};\n+\n+struct table {\n+\t/* Input parameters */\n+\tstruct rte_swx_table_params params;\n+\n+\t/* Internal. */\n+\tuint32_t key_size;\n+\tuint32_t data_size;\n+\tuint32_t key_size_shl;\n+\tuint32_t data_size_shl;\n+\tuint32_t n_buckets;\n+\tuint32_t n_buckets_ext;\n+\tuint32_t key_stack_tos;\n+\tuint32_t bkt_ext_stack_tos;\n+\tuint64_t total_size;\n+\n+\t/* Memory arrays. */\n+\tuint8_t *key_mask;\n+\tstruct bucket_extension *buckets;\n+\tstruct bucket_extension *buckets_ext;\n+\tuint8_t *keys;\n+\tuint32_t *key_stack;\n+\tuint32_t *bkt_ext_stack;\n+\tuint8_t *data;\n+};\n+\n+static inline uint8_t *\n+table_key(struct table *t, uint32_t key_id)\n+{\n+\treturn &t->keys[(uint64_t)key_id << t->key_size_shl];\n+}\n+\n+static inline uint64_t *\n+table_key_data(struct table *t, uint32_t key_id)\n+{\n+\treturn (uint64_t *)&t->data[(uint64_t)key_id << t->data_size_shl];\n+}\n+\n+static inline int\n+bkt_is_empty(struct bucket_extension *bkt)\n+{\n+\treturn (!bkt->sig[0] && !bkt->sig[1] && !bkt->sig[2] && !bkt->sig[2]) ?\n+\t\t1 : 0;\n+}\n+\n+/* Return:\n+ * 0 = Bucket key position is NOT empty;\n+ * 1 = Bucket key position is empty.\n+ */\n+static inline int\n+bkt_key_is_empty(struct bucket_extension *bkt, uint32_t bkt_pos)\n+{\n+\treturn bkt->sig[bkt_pos] ? 0 : 1;\n+}\n+\n+/* Return: 0 = Keys are NOT equal; 1 = Keys are equal. */\n+static inline int\n+bkt_keycmp(struct table *t,\n+\t struct bucket_extension *bkt,\n+\t uint8_t *input_key,\n+\t uint32_t bkt_pos,\n+\t uint32_t input_sig)\n+{\n+\tuint32_t bkt_key_id;\n+\tuint8_t *bkt_key;\n+\n+\t/* Key signature comparison. */\n+\tif (input_sig != bkt->sig[bkt_pos])\n+\t\treturn 0;\n+\n+\t/* Key comparison. */\n+\tbkt_key_id = bkt->key_id[bkt_pos];\n+\tbkt_key = table_key(t, bkt_key_id);\n+\treturn keycmp(bkt_key, input_key, t->key_mask, t->key_size);\n+}\n+\n+static inline void\n+bkt_key_install(struct table *t,\n+\t\tstruct bucket_extension *bkt,\n+\t\tstruct rte_swx_table_entry *input,\n+\t\tuint32_t bkt_pos,\n+\t\tuint32_t bkt_key_id,\n+\t\tuint32_t input_sig)\n+{\n+\tuint8_t *bkt_key;\n+\tuint64_t *bkt_data;\n+\n+\t/* Key signature. */\n+\tbkt->sig[bkt_pos] = (uint16_t)input_sig;\n+\n+\t/* Key. */\n+\tbkt->key_id[bkt_pos] = bkt_key_id;\n+\tbkt_key = table_key(t, bkt_key_id);\n+\tkeycpy(bkt_key, input->key, t->key_mask, t->key_size);\n+\n+\t/* Key data. */\n+\tbkt_data = table_key_data(t, bkt_key_id);\n+\tbkt_data[0] = input->action_id;\n+\tif (t->params.action_data_size)\n+\t\tmemcpy(&bkt_data[1],\n+\t\t input->action_data,\n+\t\t t->params.action_data_size);\n+}\n+\n+static inline void\n+bkt_key_data_update(struct table *t,\n+\t\t struct bucket_extension *bkt,\n+\t\t struct rte_swx_table_entry *input,\n+\t\t uint32_t bkt_pos)\n+{\n+\tuint32_t bkt_key_id;\n+\tuint64_t *bkt_data;\n+\n+\t/* Key. */\n+\tbkt_key_id = bkt->key_id[bkt_pos];\n+\n+\t/* Key data. */\n+\tbkt_data = table_key_data(t, bkt_key_id);\n+\tbkt_data[0] = input->action_id;\n+\tif (t->params.action_data_size)\n+\t\tmemcpy(&bkt_data[1],\n+\t\t input->action_data,\n+\t\t t->params.action_data_size);\n+}\n+\n+#define CL RTE_CACHE_LINE_ROUNDUP\n+\n+static int\n+__table_create(struct table **table,\n+\t uint64_t *memory_footprint,\n+\t struct rte_swx_table_params *params,\n+\t const char *args __rte_unused,\n+\t int numa_node)\n+{\n+\tstruct table *t;\n+\tuint8_t *memory;\n+\tsize_t table_meta_sz, key_mask_sz, bucket_sz, bucket_ext_sz, key_sz,\n+\t\tkey_stack_sz, bkt_ext_stack_sz, data_sz, total_size;\n+\tsize_t key_mask_offset, bucket_offset, bucket_ext_offset, key_offset,\n+\t\tkey_stack_offset, bkt_ext_stack_offset, data_offset;\n+\tuint32_t key_size, key_data_size, n_buckets, n_buckets_ext, i;\n+\n+\t/* Check input arguments. */\n+\tCHECK(params, EINVAL);\n+\tCHECK(params->match_type == RTE_SWX_TABLE_MATCH_EXACT, EINVAL);\n+\tCHECK(params->key_size, EINVAL);\n+\tCHECK(params->key_size <= 64, EINVAL);\n+\tCHECK(params->n_keys_max, EINVAL);\n+\n+\t/* Memory allocation. */\n+\tkey_size = rte_align64pow2(params->key_size);\n+\tif (key_size < 8)\n+\t\tkey_size = 8;\n+\tkey_data_size = rte_align64pow2(params->action_data_size + 8);\n+\tn_buckets = params->n_keys_max / KEYS_PER_BUCKET;\n+\tn_buckets_ext = params->n_keys_max / KEYS_PER_BUCKET;\n+\n+\ttable_meta_sz = CL(sizeof(struct table));\n+\tkey_mask_sz = CL(key_size);\n+\tbucket_sz = CL(n_buckets * sizeof(struct bucket_extension));\n+\tbucket_ext_sz = CL(n_buckets_ext * sizeof(struct bucket_extension));\n+\tkey_sz = CL(params->n_keys_max * key_size);\n+\tkey_stack_sz = CL(params->n_keys_max * sizeof(uint32_t));\n+\tbkt_ext_stack_sz = CL(n_buckets_ext * sizeof(uint32_t));\n+\tdata_sz = CL(params->n_keys_max * key_data_size);\n+\ttotal_size = table_meta_sz + key_mask_sz + bucket_sz + bucket_ext_sz +\n+\t\t key_sz + key_stack_sz + bkt_ext_stack_sz + data_sz;\n+\n+\tkey_mask_offset = table_meta_sz;\n+\tbucket_offset = key_mask_offset + key_mask_sz;\n+\tbucket_ext_offset = bucket_offset + bucket_sz;\n+\tkey_offset = bucket_ext_offset + bucket_ext_sz;\n+\tkey_stack_offset = key_offset + key_sz;\n+\tbkt_ext_stack_offset = key_stack_offset + key_stack_sz;\n+\tdata_offset = bkt_ext_stack_offset + bkt_ext_stack_sz;\n+\n+\tif (!table) {\n+\t\tif (memory_footprint)\n+\t\t\t*memory_footprint = total_size;\n+\t\treturn 0;\n+\t}\n+\n+\tmemory = env_malloc(total_size, RTE_CACHE_LINE_SIZE, numa_node);\n+\tCHECK(memory, ENOMEM);\n+\tmemset(memory, 0, total_size);\n+\n+\t/* Initialization. */\n+\tt = (struct table *)memory;\n+\tmemcpy(&t->params, params, sizeof(*params));\n+\n+\tt->key_size = key_size;\n+\tt->data_size = key_data_size;\n+\tt->key_size_shl = __builtin_ctzl(key_size);\n+\tt->data_size_shl = __builtin_ctzl(key_data_size);\n+\tt->n_buckets = n_buckets;\n+\tt->n_buckets_ext = n_buckets_ext;\n+\tt->total_size = total_size;\n+\n+\tt->key_mask = &memory[key_mask_offset];\n+\tt->buckets = (struct bucket_extension *)&memory[bucket_offset];\n+\tt->buckets_ext = (struct bucket_extension *)&memory[bucket_ext_offset];\n+\tt->keys = &memory[key_offset];\n+\tt->key_stack = (uint32_t *)&memory[key_stack_offset];\n+\tt->bkt_ext_stack = (uint32_t *)&memory[bkt_ext_stack_offset];\n+\tt->data = &memory[data_offset];\n+\n+\tt->params.key_mask0 = t->key_mask;\n+\n+\tif (!params->key_mask0)\n+\t\tmemset(t->key_mask, 0xFF, params->key_size);\n+\telse\n+\t\tmemcpy(t->key_mask, params->key_mask0, params->key_size);\n+\n+\tfor (i = 0; i < t->params.n_keys_max; i++)\n+\t\tt->key_stack[i] = t->params.n_keys_max - 1 - i;\n+\tt->key_stack_tos = t->params.n_keys_max;\n+\n+\tfor (i = 0; i < n_buckets_ext; i++)\n+\t\tt->bkt_ext_stack[i] = n_buckets_ext - 1 - i;\n+\tt->bkt_ext_stack_tos = n_buckets_ext;\n+\n+\t*table = t;\n+\treturn 0;\n+}\n+\n+static void\n+table_free(void *table)\n+{\n+\tstruct table *t = table;\n+\n+\tif (!t)\n+\t\treturn;\n+\n+\tenv_free(t, t->total_size);\n+}\n+\n+static int\n+table_add(void *table, struct rte_swx_table_entry *entry)\n+{\n+\tstruct table *t = table;\n+\tstruct bucket_extension *bkt0, *bkt, *bkt_prev;\n+\tuint32_t input_sig, bkt_id, i;\n+\n+\tCHECK(t, EINVAL);\n+\tCHECK(entry, EINVAL);\n+\tCHECK(entry->key, EINVAL);\n+\tCHECK((!t->params.action_data_size && !entry->action_data) ||\n+\t (t->params.action_data_size && entry->action_data), EINVAL);\n+\n+\tinput_sig = hash(entry->key, t->key_mask, t->key_size, 0);\n+\tbkt_id = input_sig & (t->n_buckets - 1);\n+\tbkt0 = &t->buckets[bkt_id];\n+\tinput_sig = (input_sig >> 16) | 1;\n+\n+\t/* Key is present in the bucket. */\n+\tfor (bkt = bkt0; bkt; bkt = bkt->next)\n+\t\tfor (i = 0; i < KEYS_PER_BUCKET; i++)\n+\t\t\tif (bkt_keycmp(t, bkt, entry->key, i, input_sig)) {\n+\t\t\t\tbkt_key_data_update(t, bkt, entry, i);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\n+\t/* Key is not present in the bucket. Bucket not full. */\n+\tfor (bkt = bkt0, bkt_prev = NULL; bkt; bkt_prev = bkt, bkt = bkt->next)\n+\t\tfor (i = 0; i < KEYS_PER_BUCKET; i++)\n+\t\t\tif (bkt_key_is_empty(bkt, i)) {\n+\t\t\t\tuint32_t new_bkt_key_id;\n+\n+\t\t\t\t/* Allocate new key & install. */\n+\t\t\t\tCHECK(t->key_stack_tos, ENOSPC);\n+\t\t\t\tnew_bkt_key_id =\n+\t\t\t\t\tt->key_stack[--t->key_stack_tos];\n+\t\t\t\tbkt_key_install(t, bkt, entry, i,\n+\t\t\t\t\t\tnew_bkt_key_id, input_sig);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\n+\t/* Bucket full: extend bucket. */\n+\tif (t->bkt_ext_stack_tos && t->key_stack_tos) {\n+\t\tstruct bucket_extension *new_bkt;\n+\t\tuint32_t new_bkt_id, new_bkt_key_id;\n+\n+\t\t/* Allocate new bucket extension & install. */\n+\t\tnew_bkt_id = t->bkt_ext_stack[--t->bkt_ext_stack_tos];\n+\t\tnew_bkt = &t->buckets_ext[new_bkt_id];\n+\t\tmemset(new_bkt, 0, sizeof(*new_bkt));\n+\t\tbkt_prev->next = new_bkt;\n+\n+\t\t/* Allocate new key & install. */\n+\t\tnew_bkt_key_id = t->key_stack[--t->key_stack_tos];\n+\t\tbkt_key_install(t, new_bkt, entry, 0,\n+\t\t\t\tnew_bkt_key_id, input_sig);\n+\t\treturn 0;\n+\t}\n+\n+\tCHECK(0, ENOSPC);\n+}\n+\n+static int\n+table_del(void *table, struct rte_swx_table_entry *entry)\n+{\n+\tstruct table *t = table;\n+\tstruct bucket_extension *bkt0, *bkt, *bkt_prev;\n+\tuint32_t input_sig, bkt_id, i;\n+\n+\tCHECK(t, EINVAL);\n+\tCHECK(entry, EINVAL);\n+\tCHECK(entry->key, EINVAL);\n+\n+\tinput_sig = hash(entry->key, t->key_mask, t->key_size, 0);\n+\tbkt_id = input_sig & (t->n_buckets - 1);\n+\tbkt0 = &t->buckets[bkt_id];\n+\tinput_sig = (input_sig >> 16) | 1;\n+\n+\t/* Key is present in the bucket. */\n+\tfor (bkt = bkt0, bkt_prev = NULL; bkt; bkt_prev = bkt, bkt = bkt->next)\n+\t\tfor (i = 0; i < KEYS_PER_BUCKET; i++)\n+\t\t\tif (bkt_keycmp(t, bkt, entry->key, i, input_sig)) {\n+\t\t\t\t/* Key free. */\n+\t\t\t\tbkt->sig[i] = 0;\n+\t\t\t\tt->key_stack[t->key_stack_tos++] =\n+\t\t\t\t\tbkt->key_id[i];\n+\n+\t\t\t\t/* Bucket extension free if empty and not the\n+\t\t\t\t * 1st in bucket.\n+\t\t\t\t */\n+\t\t\t\tif (bkt_prev && bkt_is_empty(bkt)) {\n+\t\t\t\t\tbkt_prev->next = bkt->next;\n+\t\t\t\t\tbkt_id = bkt - t->buckets_ext;\n+\t\t\t\t\tt->bkt_ext_stack[t->bkt_ext_stack_tos++]\n+\t\t\t\t\t\t= bkt_id;\n+\t\t\t\t}\n+\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\n+\treturn 0;\n+}\n+\n+static uint64_t\n+table_mailbox_size_get_unoptimized(void)\n+{\n+\treturn 0;\n+}\n+\n+static int\n+table_lookup_unoptimized(void *table,\n+\t\t\t void *mailbox __rte_unused,\n+\t\t\t uint8_t **key,\n+\t\t\t uint64_t *action_id,\n+\t\t\t uint8_t **action_data,\n+\t\t\t int *hit)\n+{\n+\tstruct table *t = table;\n+\tstruct bucket_extension *bkt0, *bkt;\n+\tuint8_t *input_key;\n+\tuint32_t input_sig, bkt_id, i;\n+\n+\tinput_key = &(*key)[t->params.key_offset];\n+\n+\tinput_sig = hash(input_key, t->key_mask, t->key_size, 0);\n+\tbkt_id = input_sig & (t->n_buckets - 1);\n+\tbkt0 = &t->buckets[bkt_id];\n+\tinput_sig = (input_sig >> 16) | 1;\n+\n+\t/* Key is present in the bucket. */\n+\tfor (bkt = bkt0; bkt; bkt = bkt->next)\n+\t\tfor (i = 0; i < KEYS_PER_BUCKET; i++)\n+\t\t\tif (bkt_keycmp(t, bkt, input_key, i, input_sig)) {\n+\t\t\t\tuint32_t bkt_key_id;\n+\t\t\t\tuint64_t *bkt_data;\n+\n+\t\t\t\t/* Key. */\n+\t\t\t\tbkt_key_id = bkt->key_id[i];\n+\n+\t\t\t\t/* Key data. */\n+\t\t\t\tbkt_data = table_key_data(t, bkt_key_id);\n+\t\t\t\t*action_id = bkt_data[0];\n+\t\t\t\t*action_data = (uint8_t *)&bkt_data[1];\n+\t\t\t\t*hit = 1;\n+\t\t\t\treturn 1;\n+\t\t\t}\n+\n+\t*hit = 0;\n+\treturn 1;\n+}\n+\n+struct mailbox {\n+\tstruct bucket_extension *bkt;\n+\tuint32_t input_sig;\n+\tuint32_t bkt_key_id;\n+\tuint32_t sig_match;\n+\tuint32_t sig_match_many;\n+\tint state;\n+};\n+\n+static uint64_t\n+table_mailbox_size_get(void)\n+{\n+\treturn sizeof(struct mailbox);\n+}\n+\n+/*\n+ * mask = match bitmask\n+ * match = at least one match\n+ * match_many = more than one match\n+ * match_pos = position of first match\n+ *\n+ *+------+-------+------------+-----------+\n+ *| mask | match | match_many | match_pos |\n+ *+------+-------+------------+-----------+\n+ *| 0000 | 0 | 0 | 00 |\n+ *| 0001 | 1 | 0 | 00 |\n+ *| 0010 | 1 | 0 | 01 |\n+ *| 0011 | 1 | 1 | 00 |\n+ *+------+-------+------------+-----------+\n+ *| 0100 | 1 | 0 | 10 |\n+ *| 0101 | 1 | 1 | 00 |\n+ *| 0110 | 1 | 1 | 01 |\n+ *| 0111 | 1 | 1 | 00 |\n+ *+------+-------+------------+-----------+\n+ *| 1000 | 1 | 0 | 11 |\n+ *| 1001 | 1 | 1 | 00 |\n+ *| 1010 | 1 | 1 | 01 |\n+ *| 1011 | 1 | 1 | 00 |\n+ *+------+-------+------------+-----------+\n+ *| 1100 | 1 | 1 | 10 |\n+ *| 1101 | 1 | 1 | 00 |\n+ *| 1110 | 1 | 1 | 01 |\n+ *| 1111 | 1 | 1 | 00 |\n+ *+------+-------+------------+-----------+\n+ *\n+ * match = 1111_1111_1111_1110 = 0xFFFE\n+ * match_many = 1111_1110_1110_1000 = 0xFEE8\n+ * match_pos = 0001_0010_0001_0011__0001_0010_0001_0000 = 0x12131210\n+ *\n+ */\n+\n+#define LUT_MATCH 0xFFFE\n+#define LUT_MATCH_MANY 0xFEE8\n+#define LUT_MATCH_POS 0x12131210\n+\n+static int\n+table_lookup(void *table,\n+\t void *mailbox,\n+\t uint8_t **key,\n+\t uint64_t *action_id,\n+\t uint8_t **action_data,\n+\t int *hit)\n+{\n+\tstruct table *t = table;\n+\tstruct mailbox *m = mailbox;\n+\n+\tswitch (m->state) {\n+\tcase 0: {\n+\t\tuint8_t *input_key = &(*key)[t->params.key_offset];\n+\t\tstruct bucket_extension *bkt;\n+\t\tuint32_t input_sig, bkt_id;\n+\n+\t\tinput_sig = hash(input_key, t->key_mask, t->key_size, 0);\n+\t\tbkt_id = input_sig & (t->n_buckets - 1);\n+\t\tbkt = &t->buckets[bkt_id];\n+\t\trte_prefetch0(bkt);\n+\n+\t\tm->bkt = bkt;\n+\t\tm->input_sig = (input_sig >> 16) | 1;\n+\t\tm->state++;\n+\t\treturn 0;\n+\t}\n+\n+\tcase 1: {\n+\t\tstruct bucket_extension *bkt = m->bkt;\n+\t\tuint32_t input_sig = m->input_sig;\n+\t\tuint32_t bkt_sig0, bkt_sig1, bkt_sig2, bkt_sig3;\n+\t\tuint32_t mask0 = 0, mask1 = 0, mask2 = 0, mask3 = 0, mask_all;\n+\t\tuint32_t sig_match = LUT_MATCH;\n+\t\tuint32_t sig_match_many = LUT_MATCH_MANY;\n+\t\tuint32_t sig_match_pos = LUT_MATCH_POS;\n+\t\tuint32_t bkt_key_id;\n+\n+\t\tbkt_sig0 = input_sig ^ bkt->sig[0];\n+\t\tif (bkt_sig0)\n+\t\t\tmask0 = 1 << 0;\n+\n+\t\tbkt_sig1 = input_sig ^ bkt->sig[1];\n+\t\tif (bkt_sig1)\n+\t\t\tmask1 = 1 << 1;\n+\n+\t\tbkt_sig2 = input_sig ^ bkt->sig[2];\n+\t\tif (bkt_sig2)\n+\t\t\tmask2 = 1 << 2;\n+\n+\t\tbkt_sig3 = input_sig ^ bkt->sig[3];\n+\t\tif (bkt_sig3)\n+\t\t\tmask3 = 1 << 3;\n+\n+\t\tmask_all = (mask0 | mask1) | (mask2 | mask3);\n+\t\tsig_match = (sig_match >> mask_all) & 1;\n+\t\tsig_match_many = (sig_match_many >> mask_all) & 1;\n+\t\tsig_match_pos = (sig_match_pos >> (mask_all << 1)) & 3;\n+\n+\t\tbkt_key_id = bkt->key_id[sig_match_pos];\n+\t\trte_prefetch0(table_key(t, bkt_key_id));\n+\t\trte_prefetch0(table_key_data(t, bkt_key_id));\n+\n+\t\tm->bkt_key_id = bkt_key_id;\n+\t\tm->sig_match = sig_match;\n+\t\tm->sig_match_many = sig_match_many;\n+\t\tm->state++;\n+\t\treturn 0;\n+\t}\n+\n+\tcase 2: {\n+\t\tuint8_t *input_key = &(*key)[t->params.key_offset];\n+\t\tstruct bucket_extension *bkt = m->bkt;\n+\t\tuint32_t bkt_key_id = m->bkt_key_id;\n+\t\tuint8_t *bkt_key = table_key(t, bkt_key_id);\n+\t\tuint64_t *bkt_data = table_key_data(t, bkt_key_id);\n+\t\tuint32_t lkp_hit;\n+\n+\t\tlkp_hit = keycmp(bkt_key, input_key, t->key_mask, t->key_size);\n+\t\tlkp_hit &= m->sig_match;\n+\t\t*action_id = bkt_data[0];\n+\t\t*action_data = (uint8_t *)&bkt_data[1];\n+\t\t*hit = lkp_hit;\n+\n+\t\tm->state = 0;\n+\n+\t\tif (!lkp_hit && (m->sig_match_many || bkt->next))\n+\t\t\treturn table_lookup_unoptimized(t,\n+\t\t\t\t\t\t\tm,\n+\t\t\t\t\t\t\tkey,\n+\t\t\t\t\t\t\taction_id,\n+\t\t\t\t\t\t\taction_data,\n+\t\t\t\t\t\t\thit);\n+\n+\t\treturn 1;\n+\t}\n+\n+\tdefault:\n+\t\treturn 0;\n+\t}\n+}\n+\n+static void *\n+table_create(struct rte_swx_table_params *params,\n+\t struct rte_swx_table_entry_list *entries,\n+\t const char *args,\n+\t int numa_node)\n+{\n+\tstruct table *t;\n+\tstruct rte_swx_table_entry *entry;\n+\tint status;\n+\n+\t/* Table create. */\n+\tstatus = __table_create(&t, NULL, params, args, numa_node);\n+\tif (status)\n+\t\treturn NULL;\n+\n+\t/* Table add entries. */\n+\tif (!entries)\n+\t\treturn t;\n+\n+\tTAILQ_FOREACH(entry, entries, node) {\n+\t\tint status;\n+\n+\t\tstatus = table_add(t, entry);\n+\t\tif (status) {\n+\t\t\ttable_free(t);\n+\t\t\treturn NULL;\n+\t\t}\n+\t}\n+\n+\treturn t;\n+}\n+\n+static uint64_t\n+table_footprint(struct rte_swx_table_params *params,\n+\t\tstruct rte_swx_table_entry_list *entries __rte_unused,\n+\t\tconst char *args)\n+{\n+\tuint64_t memory_footprint;\n+\tint status;\n+\n+\tstatus = __table_create(NULL, &memory_footprint, params, args, 0);\n+\tif (status)\n+\t\treturn 0;\n+\n+\treturn memory_footprint;\n+}\n+\n+struct rte_swx_table_ops rte_swx_table_exact_match_unoptimized_ops = {\n+\t.footprint_get = table_footprint,\n+\t.mailbox_size_get = table_mailbox_size_get_unoptimized,\n+\t.create = table_create,\n+\t.add = table_add,\n+\t.del = table_del,\n+\t.lkp = table_lookup_unoptimized,\n+\t.free = table_free,\n+};\n+\n+struct rte_swx_table_ops rte_swx_table_exact_match_ops = {\n+\t.footprint_get = table_footprint,\n+\t.mailbox_size_get = table_mailbox_size_get,\n+\t.create = table_create,\n+\t.add = table_add,\n+\t.del = table_del,\n+\t.lkp = table_lookup,\n+\t.free = table_free,\n+};\ndiff --git a/lib/librte_table/rte_swx_table_em.h b/lib/librte_table/rte_swx_table_em.h\nnew file mode 100644\nindex 000000000..909ada483\n--- /dev/null\n+++ b/lib/librte_table/rte_swx_table_em.h\n@@ -0,0 +1,30 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(c) 2020 Intel Corporation\n+ */\n+#ifndef __INCLUDE_RTE_SWX_TABLE_EM_H__\n+#define __INCLUDE_RTE_SWX_TABLE_EM_H__\n+\n+#ifdef __cplusplus\n+extern \"C\" {\n+#endif\n+\n+/**\n+ * @file\n+ * RTE SWX Exact Match Table\n+ */\n+\n+#include <stdint.h>\n+\n+#include <rte_swx_table.h>\n+\n+/** Exact match table operations - unoptimized. */\n+extern struct rte_swx_table_ops rte_swx_table_exact_match_unoptimized_ops;\n+\n+/** Exact match table operations. */\n+extern struct rte_swx_table_ops rte_swx_table_exact_match_ops;\n+\n+#ifdef __cplusplus\n+}\n+#endif\n+\n+#endif\ndiff --git a/lib/librte_table/rte_table_version.map b/lib/librte_table/rte_table_version.map\nindex 568a6c6a8..81c554b63 100644\n--- a/lib/librte_table/rte_table_version.map\n+++ b/lib/librte_table/rte_table_version.map\n@@ -18,3 +18,10 @@ DPDK_21 {\n \n \tlocal: *;\n };\n+\n+EXPERIMENTAL {\n+\tglobal:\n+\n+\trte_swx_table_exact_match_unoptimized_ops;\n+\trte_swx_table_exact_match_ops;\n+};\n", "prefixes": [ "v3", "35/41" ] }