@@ -547,7 +547,7 @@ static void *pipeline_fc_init(struct pipeline_params *params,
break;
case 16:
- table_params.ops = &rte_table_hash_key16_ext_dosig_ops;
+ table_params.ops = &rte_table_hash_key16_ext_ops;
table_params.arg_create = &table_hash_key16_params;
break;
@@ -259,13 +259,8 @@ struct rte_table_hash_key16_lru_params {
uint8_t *key_mask;
};
-/** LRU hash table operations for pre-computed key signature */
extern struct rte_table_ops rte_table_hash_key16_lru_ops;
-/** LRU hash table operations for key signature computed on lookup
- ("do-sig") */
-extern struct rte_table_ops rte_table_hash_key16_lru_dosig_ops;
-
/** Extendible bucket hash table parameters */
struct rte_table_hash_key16_ext_params {
/** Maximum number of entries (and keys) in the table */
@@ -293,13 +288,8 @@ struct rte_table_hash_key16_ext_params {
uint8_t *key_mask;
};
-/** Extendible bucket operations for pre-computed key signature */
extern struct rte_table_ops rte_table_hash_key16_ext_ops;
-/** Extendible bucket hash table operations for key signature computed on
- lookup ("do-sig") */
-extern struct rte_table_ops rte_table_hash_key16_ext_dosig_ops;
-
/**
* 32-byte key hash tables
*
@@ -610,19 +610,6 @@ rte_table_hash_entry_delete_key16_ext(
#define lookup1_stage1(mbuf1, bucket1, f) \
{ \
- uint64_t signature; \
- uint32_t bucket_index; \
- \
- signature = RTE_MBUF_METADATA_UINT32(mbuf1, f->signature_offset);\
- bucket_index = signature & (f->n_buckets - 1); \
- bucket1 = (struct rte_bucket_4_16 *) \
- &f->memory[bucket_index * f->bucket_size]; \
- rte_prefetch0(bucket1); \
- rte_prefetch0((void *)(((uintptr_t) bucket1) + RTE_CACHE_LINE_SIZE));\
-}
-
-#define lookup1_stage1_dosig(mbuf1, bucket1, f) \
-{ \
uint64_t *key; \
uint64_t signature = 0; \
uint32_t bucket_index; \
@@ -775,26 +762,6 @@ rte_table_hash_entry_delete_key16_ext(
#define lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f) \
{ \
- uint64_t signature10, signature11; \
- uint32_t bucket10_index, bucket11_index; \
- \
- signature10 = RTE_MBUF_METADATA_UINT32(mbuf10, f->signature_offset);\
- bucket10_index = signature10 & (f->n_buckets - 1); \
- bucket10 = (struct rte_bucket_4_16 *) \
- &f->memory[bucket10_index * f->bucket_size]; \
- rte_prefetch0(bucket10); \
- rte_prefetch0((void *)(((uintptr_t) bucket10) + RTE_CACHE_LINE_SIZE));\
- \
- signature11 = RTE_MBUF_METADATA_UINT32(mbuf11, f->signature_offset);\
- bucket11_index = signature11 & (f->n_buckets - 1); \
- bucket11 = (struct rte_bucket_4_16 *) \
- &f->memory[bucket11_index * f->bucket_size]; \
- rte_prefetch0(bucket11); \
- rte_prefetch0((void *)(((uintptr_t) bucket11) + RTE_CACHE_LINE_SIZE));\
-}
-
-#define lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f) \
-{ \
uint64_t *key10, *key11; \
uint64_t hash_offset_buffer[2]; \
uint64_t signature10, signature11; \
@@ -916,6 +883,7 @@ rte_table_hash_lookup_key16_lru(
uint64_t pkts_mask_out = 0;
__rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
+
RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in);
/* Cannot run the pipeline with less than 5 packets */
@@ -932,8 +900,8 @@ rte_table_hash_lookup_key16_lru(
}
*lookup_hit_mask = pkts_mask_out;
- RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f,
- n_pkts_in - __builtin_popcountll(pkts_mask_out));
+ RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in -
+ __builtin_popcountll(pkts_mask_out));
return 0;
}
@@ -1029,135 +997,6 @@ rte_table_hash_lookup_key16_lru(
} /* rte_table_hash_lookup_key16_lru() */
static int
-rte_table_hash_lookup_key16_lru_dosig(
- void *table,
- struct rte_mbuf **pkts,
- uint64_t pkts_mask,
- uint64_t *lookup_hit_mask,
- void **entries)
-{
- struct rte_table_hash *f = (struct rte_table_hash *) table;
- struct rte_bucket_4_16 *bucket10, *bucket11, *bucket20, *bucket21;
- struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21;
- uint32_t pkt00_index, pkt01_index, pkt10_index;
- uint32_t pkt11_index, pkt20_index, pkt21_index;
- uint64_t pkts_mask_out = 0;
-
- __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
-
- RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in);
-
- /* Cannot run the pipeline with less than 5 packets */
- if (__builtin_popcountll(pkts_mask) < 5) {
- for ( ; pkts_mask; ) {
- struct rte_bucket_4_16 *bucket;
- struct rte_mbuf *mbuf;
- uint32_t pkt_index;
-
- lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f);
- lookup1_stage1_dosig(mbuf, bucket, f);
- lookup1_stage2_lru(pkt_index, mbuf, bucket,
- pkts_mask_out, entries, f);
- }
-
- *lookup_hit_mask = pkts_mask_out;
- RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in -
- __builtin_popcountll(pkts_mask_out));
- return 0;
- }
-
- /*
- * Pipeline fill
- *
- */
- /* Pipeline stage 0 */
- lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
- pkts_mask, f);
-
- /* Pipeline feed */
- mbuf10 = mbuf00;
- mbuf11 = mbuf01;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 0 */
- lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
- pkts_mask, f);
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
-
- /*
- * Pipeline run
- *
- */
- for ( ; pkts_mask; ) {
- /* Pipeline feed */
- bucket20 = bucket10;
- bucket21 = bucket11;
- mbuf20 = mbuf10;
- mbuf21 = mbuf11;
- mbuf10 = mbuf00;
- mbuf11 = mbuf01;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 0 */
- lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,
- mbuf00, mbuf01, pkts, pkts_mask, f);
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
-
- /* Pipeline stage 2 */
- lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21,
- bucket20, bucket21, pkts_mask_out, entries, f);
- }
-
- /*
- * Pipeline flush
- *
- */
- /* Pipeline feed */
- bucket20 = bucket10;
- bucket21 = bucket11;
- mbuf20 = mbuf10;
- mbuf21 = mbuf11;
- mbuf10 = mbuf00;
- mbuf11 = mbuf01;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
-
- /* Pipeline stage 2 */
- lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21,
- bucket20, bucket21, pkts_mask_out, entries, f);
-
- /* Pipeline feed */
- bucket20 = bucket10;
- bucket21 = bucket11;
- mbuf20 = mbuf10;
- mbuf21 = mbuf11;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
-
- /* Pipeline stage 2 */
- lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21,
- bucket20, bucket21, pkts_mask_out, entries, f);
-
- *lookup_hit_mask = pkts_mask_out;
- RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in -
- __builtin_popcountll(pkts_mask_out));
- return 0;
-} /* rte_table_hash_lookup_key16_lru_dosig() */
-
-static int
rte_table_hash_lookup_key16_ext(
void *table,
struct rte_mbuf **pkts,
@@ -1175,6 +1014,7 @@ rte_table_hash_lookup_key16_ext(
uint64_t *keys[RTE_PORT_IN_BURST_SIZE_MAX];
__rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
+
RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in);
/* Cannot run the pipeline with less than 5 packets */
@@ -1309,158 +1149,6 @@ rte_table_hash_lookup_key16_ext(
} /* rte_table_hash_lookup_key16_ext() */
static int
-rte_table_hash_lookup_key16_ext_dosig(
- void *table,
- struct rte_mbuf **pkts,
- uint64_t pkts_mask,
- uint64_t *lookup_hit_mask,
- void **entries)
-{
- struct rte_table_hash *f = (struct rte_table_hash *) table;
- struct rte_bucket_4_16 *bucket10, *bucket11, *bucket20, *bucket21;
- struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21;
- uint32_t pkt00_index, pkt01_index, pkt10_index;
- uint32_t pkt11_index, pkt20_index, pkt21_index;
- uint64_t pkts_mask_out = 0, buckets_mask = 0;
- struct rte_bucket_4_16 *buckets[RTE_PORT_IN_BURST_SIZE_MAX];
- uint64_t *keys[RTE_PORT_IN_BURST_SIZE_MAX];
-
- __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
-
- RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in);
-
- /* Cannot run the pipeline with less than 5 packets */
- if (__builtin_popcountll(pkts_mask) < 5) {
- for ( ; pkts_mask; ) {
- struct rte_bucket_4_16 *bucket;
- struct rte_mbuf *mbuf;
- uint32_t pkt_index;
-
- lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f);
- lookup1_stage1_dosig(mbuf, bucket, f);
- lookup1_stage2_ext(pkt_index, mbuf, bucket,
- pkts_mask_out, entries, buckets_mask,
- buckets, keys, f);
- }
-
- goto grind_next_buckets;
- }
-
- /*
- * Pipeline fill
- *
- */
- /* Pipeline stage 0 */
- lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
- pkts_mask, f);
-
- /* Pipeline feed */
- mbuf10 = mbuf00;
- mbuf11 = mbuf01;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 0 */
- lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
- pkts_mask, f);
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
-
- /*
- * Pipeline run
- *
- */
- for ( ; pkts_mask; ) {
- /* Pipeline feed */
- bucket20 = bucket10;
- bucket21 = bucket11;
- mbuf20 = mbuf10;
- mbuf21 = mbuf11;
- mbuf10 = mbuf00;
- mbuf11 = mbuf01;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 0 */
- lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,
- mbuf00, mbuf01, pkts, pkts_mask, f);
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
-
- /* Pipeline stage 2 */
- lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21,
- bucket20, bucket21, pkts_mask_out, entries,
- buckets_mask, buckets, keys, f);
- }
-
- /*
- * Pipeline flush
- *
- */
- /* Pipeline feed */
- bucket20 = bucket10;
- bucket21 = bucket11;
- mbuf20 = mbuf10;
- mbuf21 = mbuf11;
- mbuf10 = mbuf00;
- mbuf11 = mbuf01;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
-
- /* Pipeline stage 2 */
- lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21,
- bucket20, bucket21, pkts_mask_out, entries,
- buckets_mask, buckets, keys, f);
-
- /* Pipeline feed */
- bucket20 = bucket10;
- bucket21 = bucket11;
- mbuf20 = mbuf10;
- mbuf21 = mbuf11;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
-
- /* Pipeline stage 2 */
- lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21,
- bucket20, bucket21, pkts_mask_out, entries,
- buckets_mask, buckets, keys, f);
-
-grind_next_buckets:
- /* Grind next buckets */
- for ( ; buckets_mask; ) {
- uint64_t buckets_mask_next = 0;
-
- for ( ; buckets_mask; ) {
- uint64_t pkt_mask;
- uint32_t pkt_index;
-
- pkt_index = __builtin_ctzll(buckets_mask);
- pkt_mask = 1LLU << pkt_index;
- buckets_mask &= ~pkt_mask;
-
- lookup_grinder(pkt_index, buckets, keys, pkts_mask_out,
- entries, buckets_mask_next, f);
- }
-
- buckets_mask = buckets_mask_next;
- }
-
- *lookup_hit_mask = pkts_mask_out;
- RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in -
- __builtin_popcountll(pkts_mask_out));
- return 0;
-} /* rte_table_hash_lookup_key16_ext_dosig() */
-
-static int
rte_table_hash_key16_stats_read(void *table, struct rte_table_stats *stats, int clear)
{
struct rte_table_hash *t = table;
@@ -1479,37 +1167,15 @@ struct rte_table_ops rte_table_hash_key16_lru_ops = {
.f_free = rte_table_hash_free_key16_lru,
.f_add = rte_table_hash_entry_add_key16_lru,
.f_delete = rte_table_hash_entry_delete_key16_lru,
- .f_add_bulk = NULL,
- .f_delete_bulk = NULL,
.f_lookup = rte_table_hash_lookup_key16_lru,
.f_stats = rte_table_hash_key16_stats_read,
};
-struct rte_table_ops rte_table_hash_key16_lru_dosig_ops = {
- .f_create = rte_table_hash_create_key16_lru,
- .f_free = rte_table_hash_free_key16_lru,
- .f_add = rte_table_hash_entry_add_key16_lru,
- .f_delete = rte_table_hash_entry_delete_key16_lru,
- .f_lookup = rte_table_hash_lookup_key16_lru_dosig,
- .f_stats = rte_table_hash_key16_stats_read,
-};
-
struct rte_table_ops rte_table_hash_key16_ext_ops = {
.f_create = rte_table_hash_create_key16_ext,
.f_free = rte_table_hash_free_key16_ext,
.f_add = rte_table_hash_entry_add_key16_ext,
.f_delete = rte_table_hash_entry_delete_key16_ext,
- .f_add_bulk = NULL,
- .f_delete_bulk = NULL,
.f_lookup = rte_table_hash_lookup_key16_ext,
.f_stats = rte_table_hash_key16_stats_read,
};
-
-struct rte_table_ops rte_table_hash_key16_ext_dosig_ops = {
- .f_create = rte_table_hash_create_key16_ext,
- .f_free = rte_table_hash_free_key16_ext,
- .f_add = rte_table_hash_entry_add_key16_ext,
- .f_delete = rte_table_hash_entry_delete_key16_ext,
- .f_lookup = rte_table_hash_lookup_key16_ext_dosig,
- .f_stats = rte_table_hash_key16_stats_read,
-};
@@ -292,7 +292,7 @@ app_main_loop_worker_pipeline_hash(void) {
};
struct rte_pipeline_table_params table_params = {
- .ops = &rte_table_hash_key16_ext_dosig_ops,
+ .ops = &rte_table_hash_key16_ext_ops,
.arg_create = &table_hash_params,
.f_action_hit = NULL,
.f_action_miss = NULL,
@@ -317,7 +317,7 @@ app_main_loop_worker_pipeline_hash(void) {
};
struct rte_pipeline_table_params table_params = {
- .ops = &rte_table_hash_key16_lru_dosig_ops,
+ .ops = &rte_table_hash_key16_lru_ops,
.arg_create = &table_hash_params,
.f_action_hit = NULL,
.f_action_miss = NULL,
@@ -528,7 +528,7 @@ test_table_hash16lru(void)
table_packets.n_hit_packets = 50;
table_packets.n_miss_packets = 50;
- status = test_table_type(&rte_table_hash_key16_lru_dosig_ops,
+ status = test_table_type(&rte_table_hash_key16_lru_ops,
(void *)&key16lru_params, (void *)key16lru, &table_packets,
NULL, 0);
VERIFY(status, CHECK_TABLE_OK);
@@ -536,7 +536,7 @@ test_table_hash16lru(void)
/* Invalid parameters */
key16lru_params.n_entries = 0;
- status = test_table_type(&rte_table_hash_key16_lru_dosig_ops,
+ status = test_table_type(&rte_table_hash_key16_lru_ops,
(void *)&key16lru_params, (void *)key16lru, &table_packets,
NULL, 0);
VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
@@ -544,7 +544,7 @@ test_table_hash16lru(void)
key16lru_params.n_entries = 1<<16;
key16lru_params.f_hash = NULL;
- status = test_table_type(&rte_table_hash_key16_lru_dosig_ops,
+ status = test_table_type(&rte_table_hash_key16_lru_ops,
(void *)&key16lru_params, (void *)key16lru, &table_packets,
NULL, 0);
VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
@@ -713,7 +713,7 @@ test_table_hash16ext(void)
table_packets.n_hit_packets = 50;
table_packets.n_miss_packets = 50;
- status = test_table_type(&rte_table_hash_key16_ext_dosig_ops,
+ status = test_table_type(&rte_table_hash_key16_ext_ops,
(void *)&key16ext_params, (void *)key16ext, &table_packets,
NULL, 0);
VERIFY(status, CHECK_TABLE_OK);
@@ -721,7 +721,7 @@ test_table_hash16ext(void)
/* Invalid parameters */
key16ext_params.n_entries = 0;
- status = test_table_type(&rte_table_hash_key16_ext_dosig_ops,
+ status = test_table_type(&rte_table_hash_key16_ext_ops,
(void *)&key16ext_params, (void *)key16ext, &table_packets,
NULL, 0);
VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
@@ -729,7 +729,7 @@ test_table_hash16ext(void)
key16ext_params.n_entries = 1<<16;
key16ext_params.f_hash = NULL;
- status = test_table_type(&rte_table_hash_key16_ext_dosig_ops,
+ status = test_table_type(&rte_table_hash_key16_ext_ops,
(void *)&key16ext_params, (void *)key16ext, &table_packets,
NULL, 0);
VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
@@ -737,7 +737,7 @@ test_table_hash16ext(void)
key16ext_params.f_hash = pipeline_test_hash;
key16ext_params.n_entries_ext = 0;
- status = test_table_type(&rte_table_hash_key16_ext_dosig_ops,
+ status = test_table_type(&rte_table_hash_key16_ext_ops,
(void *)&key16ext_params, (void *)key16ext, &table_packets, NULL, 0);
VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
@@ -904,7 +904,7 @@ test_table_hash_lru(void)
if (status < 0)
return status;
- status = test_table_hash_lru_generic(&rte_table_hash_key16_lru_dosig_ops);
+ status = test_table_hash_lru_generic(&rte_table_hash_key16_lru_ops);
if (status < 0)
return status;
@@ -933,7 +933,7 @@ test_table_hash_ext(void)
if (status < 0)
return status;
- status = test_table_hash_ext_generic(&rte_table_hash_key16_ext_dosig_ops);
+ status = test_table_hash_ext_generic(&rte_table_hash_key16_ext_ops);
if (status < 0)
return status;