[dpdk-dev] [PATCH v3 08/18] librte_table: rework variable size key ext hash tables
Cristian Dumitrescu
cristian.dumitrescu at intel.com
Wed Oct 18 17:03:25 CEST 2017
Rework for the variable size key extendible bucket (EXT) hash
table to use the mask-based hash function and the unified
parameter structure.
Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu at intel.com>
---
.../pipeline/pipeline_flow_classification_be.c | 12 +-
lib/librte_table/rte_table_hash.h | 34 ----
lib/librte_table/rte_table_hash_ext.c | 179 ++++++++++++++-------
test/test-pipeline/pipeline_hash.c | 10 +-
4 files changed, 130 insertions(+), 105 deletions(-)
mode change 100644 => 100755 test/test-pipeline/pipeline_hash.c
diff --git a/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c b/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c
index e131a5b..4a4007c 100755
--- a/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c
+++ b/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c
@@ -516,16 +516,16 @@ static void *pipeline_fc_init(struct pipeline_params *params,
.seed = 0,
};
- struct rte_table_hash_ext_params
- table_hash_params = {
+ struct rte_table_hash_params table_hash_params = {
+ .name = p->name,
.key_size = p_fc->key_size,
+ .key_offset = p_fc->key_offset,
+ .key_mask = (p_fc->key_mask_present) ?
+ p_fc->key_mask : NULL,
.n_keys = p_fc->n_flows,
.n_buckets = p_fc->n_flows / 4,
- .n_buckets_ext = p_fc->n_flows / 4,
- .f_hash = hash_func[(p_fc->key_size / 8) - 1],
+ .f_hash = (rte_table_hash_op_hash)hash_func[(p_fc->key_size / 8) - 1],
.seed = 0,
- .signature_offset = p_fc->hash_offset,
- .key_offset = p_fc->key_offset,
};
struct rte_pipeline_table_params table_params = {
diff --git a/lib/librte_table/rte_table_hash.h b/lib/librte_table/rte_table_hash.h
index bb5b83d..0024b99 100755
--- a/lib/librte_table/rte_table_hash.h
+++ b/lib/librte_table/rte_table_hash.h
@@ -135,40 +135,6 @@ typedef uint64_t (*rte_table_hash_op_hash_nomask)(
uint32_t key_size,
uint64_t seed);
-/**
- * Hash tables with configurable key size
- *
- */
-/** Extendible bucket hash table parameters */
-struct rte_table_hash_ext_params {
- /** Key size (number of bytes) */
- uint32_t key_size;
-
- /** Maximum number of keys */
- uint32_t n_keys;
-
- /** Number of hash table buckets. Each bucket stores up to 4 keys. */
- uint32_t n_buckets;
-
- /** Number of hash table bucket extensions. Each bucket extension has
- space for 4 keys and each bucket can have 0, 1 or more extensions. */
- uint32_t n_buckets_ext;
-
- /** Hash function */
- rte_table_hash_op_hash_nomask f_hash;
-
- /** Seed value for the hash function */
- uint64_t seed;
-
- /** Byte offset within packet meta-data where the 4-byte key signature
- is located. Valid for pre-computed key signature tables, ignored for
- do-sig tables. */
- uint32_t signature_offset;
-
- /** Byte offset within packet meta-data where the key is located */
- uint32_t key_offset;
-};
-
extern struct rte_table_ops rte_table_hash_ext_ops;
/** LRU hash table parameters */
diff --git a/lib/librte_table/rte_table_hash_ext.c b/lib/librte_table/rte_table_hash_ext.c
index 72802b8..0a743ad 100755
--- a/lib/librte_table/rte_table_hash_ext.c
+++ b/lib/librte_table/rte_table_hash_ext.c
@@ -104,9 +104,8 @@ struct rte_table_hash {
uint32_t n_keys;
uint32_t n_buckets;
uint32_t n_buckets_ext;
- rte_table_hash_op_hash_nomask f_hash;
+ rte_table_hash_op_hash f_hash;
uint64_t seed;
- uint32_t signature_offset;
uint32_t key_offset;
/* Internal */
@@ -120,6 +119,7 @@ struct rte_table_hash {
struct grinder grinders[RTE_PORT_IN_BURST_SIZE_MAX];
/* Tables */
+ uint64_t *key_mask;
struct bucket *buckets;
struct bucket *buckets_ext;
uint8_t *key_mem;
@@ -132,29 +132,53 @@ struct rte_table_hash {
};
static int
-check_params_create(struct rte_table_hash_ext_params *params)
+keycmp(void *a, void *b, void *b_mask, uint32_t n_bytes)
{
- uint32_t n_buckets_min;
+ uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask;
+ uint32_t i;
+
+ for (i = 0; i < n_bytes / sizeof(uint64_t); i++)
+ if (a64[i] != (b64[i] & b_mask64[i]))
+ return 1;
+
+ return 0;
+}
+
+static void
+keycpy(void *dst, void *src, void *src_mask, uint32_t n_bytes)
+{
+ uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask;
+ uint32_t i;
+
+ for (i = 0; i < n_bytes / sizeof(uint64_t); i++)
+ dst64[i] = src64[i] & src_mask64[i];
+}
+
+static int
+check_params_create(struct rte_table_hash_params *params)
+{
+ /* name */
+ if (params->name == NULL) {
+ RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__);
+ return -EINVAL;
+ }
/* key_size */
- if ((params->key_size == 0) ||
+ if ((params->key_size < sizeof(uint64_t)) ||
(!rte_is_power_of_2(params->key_size))) {
RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__);
return -EINVAL;
}
/* n_keys */
- if ((params->n_keys == 0) ||
- (!rte_is_power_of_2(params->n_keys))) {
+ if (params->n_keys == 0) {
RTE_LOG(ERR, TABLE, "%s: n_keys invalid value\n", __func__);
return -EINVAL;
}
/* n_buckets */
- n_buckets_min = (params->n_keys + KEYS_PER_BUCKET - 1) / params->n_keys;
if ((params->n_buckets == 0) ||
- (!rte_is_power_of_2(params->n_keys)) ||
- (params->n_buckets < n_buckets_min)) {
+ (!rte_is_power_of_2(params->n_buckets))) {
RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__);
return -EINVAL;
}
@@ -171,15 +195,13 @@ check_params_create(struct rte_table_hash_ext_params *params)
static void *
rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size)
{
- struct rte_table_hash_ext_params *p =
- params;
+ struct rte_table_hash_params *p = params;
struct rte_table_hash *t;
- uint32_t total_size, table_meta_sz;
- uint32_t bucket_sz, bucket_ext_sz, key_sz;
- uint32_t key_stack_sz, bkt_ext_stack_sz, data_sz;
- uint32_t bucket_offset, bucket_ext_offset, key_offset;
- uint32_t key_stack_offset, bkt_ext_stack_offset, data_offset;
- uint32_t i;
+ uint64_t table_meta_sz, key_mask_sz, bucket_sz, bucket_ext_sz, key_sz;
+ uint64_t key_stack_sz, bkt_ext_stack_sz, data_sz, total_size;
+ uint64_t key_mask_offset, bucket_offset, bucket_ext_offset, key_offset;
+ uint64_t key_stack_offset, bkt_ext_stack_offset, data_offset;
+ uint32_t n_buckets_ext, i;
/* Check input parameters */
if ((check_params_create(p) != 0) ||
@@ -188,38 +210,66 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size)
(sizeof(struct bucket) != (RTE_CACHE_LINE_SIZE / 2)))
return NULL;
+ /*
+ * Table dimensioning
+ *
+ * Objective: Pick the number of bucket extensions (n_buckets_ext) so that
+ * it is guaranteed that n_keys keys can be stored in the table at any time.
+ *
+ * The worst case scenario takes place when all the n_keys keys fall into
+ * the same bucket. Actually, due to the KEYS_PER_BUCKET scheme, the worst
+ * case takes place when (n_keys - KEYS_PER_BUCKET + 1) keys fall into the
+ * same bucket, while the remaining (KEYS_PER_BUCKET - 1) keys each fall
+ * into a different bucket. This case defeats the purpose of the hash table.
+ * It indicates unsuitable f_hash or n_keys to n_buckets ratio.
+ *
+ * n_buckets_ext = n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1
+ */
+ n_buckets_ext = p->n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1;
+
/* Memory allocation */
table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash));
+ key_mask_sz = RTE_CACHE_LINE_ROUNDUP(p->key_size);
bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket));
bucket_ext_sz =
- RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(struct bucket));
+ RTE_CACHE_LINE_ROUNDUP(n_buckets_ext * sizeof(struct bucket));
key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size);
key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t));
bkt_ext_stack_sz =
- RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(uint32_t));
+ RTE_CACHE_LINE_ROUNDUP(n_buckets_ext * sizeof(uint32_t));
data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
- total_size = table_meta_sz + bucket_sz + bucket_ext_sz + key_sz +
- key_stack_sz + bkt_ext_stack_sz + data_sz;
+ total_size = table_meta_sz + key_mask_sz + bucket_sz + bucket_ext_sz +
+ key_sz + key_stack_sz + bkt_ext_stack_sz + data_sz;
- t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
+ if (total_size > SIZE_MAX) {
+ RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes"
+ " for hash table %s\n",
+ __func__, total_size, p->name);
+ return NULL;
+ }
+
+ t = rte_zmalloc_socket(p->name,
+ (size_t)total_size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
if (t == NULL) {
- RTE_LOG(ERR, TABLE,
- "%s: Cannot allocate %u bytes for hash table\n",
- __func__, total_size);
+ RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes"
+ " for hash table %s\n",
+ __func__, total_size, p->name);
return NULL;
}
- RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table memory footprint is "
- "%u bytes\n", __func__, p->key_size, total_size);
+ RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table %s memory "
+ "footprint is %" PRIu64 " bytes\n",
+ __func__, p->key_size, p->name, total_size);
/* Memory initialization */
t->key_size = p->key_size;
t->entry_size = entry_size;
t->n_keys = p->n_keys;
t->n_buckets = p->n_buckets;
- t->n_buckets_ext = p->n_buckets_ext;
+ t->n_buckets_ext = n_buckets_ext;
t->f_hash = p->f_hash;
t->seed = p->seed;
- t->signature_offset = p->signature_offset;
t->key_offset = p->key_offset;
/* Internal */
@@ -228,13 +278,15 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size)
t->data_size_shl = __builtin_ctzl(entry_size);
/* Tables */
- bucket_offset = 0;
+ key_mask_offset = 0;
+ bucket_offset = key_mask_offset + key_mask_sz;
bucket_ext_offset = bucket_offset + bucket_sz;
key_offset = bucket_ext_offset + bucket_ext_sz;
key_stack_offset = key_offset + key_sz;
bkt_ext_stack_offset = key_stack_offset + key_stack_sz;
data_offset = bkt_ext_stack_offset + bkt_ext_stack_sz;
+ t->key_mask = (uint64_t *) &t->memory[key_mask_offset];
t->buckets = (struct bucket *) &t->memory[bucket_offset];
t->buckets_ext = (struct bucket *) &t->memory[bucket_ext_offset];
t->key_mem = &t->memory[key_offset];
@@ -242,6 +294,12 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size)
t->bkt_ext_stack = (uint32_t *) &t->memory[bkt_ext_stack_offset];
t->data_mem = &t->memory[data_offset];
+ /* Key mask */
+ if (p->key_mask == NULL)
+ memset(t->key_mask, 0xFF, p->key_size);
+ else
+ memcpy(t->key_mask, p->key_mask, p->key_size);
+
/* Key stack */
for (i = 0; i < t->n_keys; i++)
t->key_stack[i] = t->n_keys - 1 - i;
@@ -277,7 +335,7 @@ rte_table_hash_ext_entry_add(void *table, void *key, void *entry,
uint64_t sig;
uint32_t bkt_index, i;
- sig = t->f_hash(key, t->key_size, t->seed);
+ sig = t->f_hash(key, t->key_mask, t->key_size, t->seed);
bkt_index = sig & t->bucket_mask;
bkt0 = &t->buckets[bkt_index];
sig = (sig >> 16) | 1LLU;
@@ -290,7 +348,7 @@ rte_table_hash_ext_entry_add(void *table, void *key, void *entry,
uint8_t *bkt_key =
&t->key_mem[bkt_key_index << t->key_size_shl];
- if ((sig == bkt_sig) && (memcmp(key, bkt_key,
+ if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask,
t->key_size) == 0)) {
uint8_t *data = &t->data_mem[bkt_key_index <<
t->data_size_shl];
@@ -327,7 +385,7 @@ rte_table_hash_ext_entry_add(void *table, void *key, void *entry,
bkt->sig[i] = (uint16_t) sig;
bkt->key_pos[i] = bkt_key_index;
- memcpy(bkt_key, key, t->key_size);
+ keycpy(bkt_key, key, t->key_mask, t->key_size);
memcpy(data, entry, t->entry_size);
*key_found = 0;
@@ -358,7 +416,7 @@ rte_table_hash_ext_entry_add(void *table, void *key, void *entry,
/* Install new key into bucket */
bkt->sig[0] = (uint16_t) sig;
bkt->key_pos[0] = bkt_key_index;
- memcpy(bkt_key, key, t->key_size);
+ keycpy(bkt_key, key, t->key_mask, t->key_size);
memcpy(data, entry, t->entry_size);
*key_found = 0;
@@ -378,7 +436,7 @@ void *entry)
uint64_t sig;
uint32_t bkt_index, i;
- sig = t->f_hash(key, t->key_size, t->seed);
+ sig = t->f_hash(key, t->key_mask, t->key_size, t->seed);
bkt_index = sig & t->bucket_mask;
bkt0 = &t->buckets[bkt_index];
sig = (sig >> 16) | 1LLU;
@@ -392,7 +450,7 @@ void *entry)
uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
t->key_size_shl];
- if ((sig == bkt_sig) && (memcmp(key, bkt_key,
+ if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask,
t->key_size) == 0)) {
uint8_t *data = &t->data_mem[bkt_key_index <<
t->data_size_shl];
@@ -457,7 +515,7 @@ static int rte_table_hash_ext_lookup_unoptimized(
pkt = pkts[pkt_index];
key = RTE_MBUF_METADATA_UINT8_PTR(pkt, t->key_offset);
- sig = (uint64_t) t->f_hash(key, t->key_size, t->seed);
+ sig = (uint64_t) t->f_hash(key, t->key_mask, t->key_size, t->seed);
bkt_index = sig & t->bucket_mask;
bkt0 = &t->buckets[bkt_index];
@@ -471,8 +529,8 @@ static int rte_table_hash_ext_lookup_unoptimized(
uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
t->key_size_shl];
- if ((sig == bkt_sig) && (memcmp(key, bkt_key,
- t->key_size) == 0)) {
+ if ((sig == bkt_sig) && (keycmp(bkt_key, key,
+ t->key_mask, t->key_size) == 0)) {
uint8_t *data = &t->data_mem[
bkt_key_index << t->data_size_shl];
@@ -571,11 +629,12 @@ static int rte_table_hash_ext_lookup_unoptimized(
{ \
uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(mbuf, f->key_offset);\
uint64_t *bkt_key = (uint64_t *) key; \
+ uint64_t *key_mask = f->key_mask; \
\
switch (f->key_size) { \
case 8: \
{ \
- uint64_t xor = pkt_key[0] ^ bkt_key[0]; \
+ uint64_t xor = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \
match_key = 0; \
if (xor == 0) \
match_key = 1; \
@@ -586,8 +645,8 @@ static int rte_table_hash_ext_lookup_unoptimized(
{ \
uint64_t xor[2], or; \
\
- xor[0] = pkt_key[0] ^ bkt_key[0]; \
- xor[1] = pkt_key[1] ^ bkt_key[1]; \
+ xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \
+ xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \
or = xor[0] | xor[1]; \
match_key = 0; \
if (or == 0) \
@@ -599,10 +658,10 @@ static int rte_table_hash_ext_lookup_unoptimized(
{ \
uint64_t xor[4], or; \
\
- xor[0] = pkt_key[0] ^ bkt_key[0]; \
- xor[1] = pkt_key[1] ^ bkt_key[1]; \
- xor[2] = pkt_key[2] ^ bkt_key[2]; \
- xor[3] = pkt_key[3] ^ bkt_key[3]; \
+ xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \
+ xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \
+ xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2]; \
+ xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3]; \
or = xor[0] | xor[1] | xor[2] | xor[3]; \
match_key = 0; \
if (or == 0) \
@@ -614,14 +673,14 @@ static int rte_table_hash_ext_lookup_unoptimized(
{ \
uint64_t xor[8], or; \
\
- xor[0] = pkt_key[0] ^ bkt_key[0]; \
- xor[1] = pkt_key[1] ^ bkt_key[1]; \
- xor[2] = pkt_key[2] ^ bkt_key[2]; \
- xor[3] = pkt_key[3] ^ bkt_key[3]; \
- xor[4] = pkt_key[4] ^ bkt_key[4]; \
- xor[5] = pkt_key[5] ^ bkt_key[5]; \
- xor[6] = pkt_key[6] ^ bkt_key[6]; \
- xor[7] = pkt_key[7] ^ bkt_key[7]; \
+ xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \
+ xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \
+ xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2]; \
+ xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3]; \
+ xor[4] = (pkt_key[4] & key_mask[4]) ^ bkt_key[4]; \
+ xor[5] = (pkt_key[5] & key_mask[5]) ^ bkt_key[5]; \
+ xor[6] = (pkt_key[6] & key_mask[6]) ^ bkt_key[6]; \
+ xor[7] = (pkt_key[7] & key_mask[7]) ^ bkt_key[7]; \
or = xor[0] | xor[1] | xor[2] | xor[3] | \
xor[4] | xor[5] | xor[6] | xor[7]; \
match_key = 0; \
@@ -632,7 +691,7 @@ static int rte_table_hash_ext_lookup_unoptimized(
\
default: \
match_key = 0; \
- if (memcmp(pkt_key, bkt_key, f->key_size) == 0) \
+ if (keycmp(bkt_key, pkt_key, key_mask, f->key_size) == 0) \
match_key = 1; \
} \
}
@@ -688,20 +747,20 @@ static int rte_table_hash_ext_lookup_unoptimized(
struct bucket *bkt10, *bkt11, *buckets = t->buckets; \
uint8_t *key10, *key11; \
uint64_t bucket_mask = t->bucket_mask; \
- rte_table_hash_op_hash_nomask f_hash = t->f_hash; \
+ rte_table_hash_op_hash f_hash = t->f_hash; \
uint64_t seed = t->seed; \
uint32_t key_size = t->key_size; \
uint32_t key_offset = t->key_offset; \
\
mbuf10 = pkts[pkt10_index]; \
key10 = RTE_MBUF_METADATA_UINT8_PTR(mbuf10, key_offset); \
- sig10 = (uint64_t) f_hash(key10, key_size, seed); \
+ sig10 = (uint64_t) f_hash(key10, t->key_mask, key_size, seed); \
bkt10_index = sig10 & bucket_mask; \
bkt10 = &buckets[bkt10_index]; \
\
mbuf11 = pkts[pkt11_index]; \
key11 = RTE_MBUF_METADATA_UINT8_PTR(mbuf11, key_offset); \
- sig11 = (uint64_t) f_hash(key11, key_size, seed); \
+ sig11 = (uint64_t) f_hash(key11, t->key_mask, key_size, seed); \
bkt11_index = sig11 & bucket_mask; \
bkt11 = &buckets[bkt11_index]; \
\
@@ -969,7 +1028,7 @@ rte_table_hash_ext_stats_read(void *table, struct rte_table_stats *stats, int cl
return 0;
}
-struct rte_table_ops rte_table_hash_ext_ops = {
+struct rte_table_ops rte_table_hash_ext_ops = {
.f_create = rte_table_hash_ext_create,
.f_free = rte_table_hash_ext_free,
.f_add = rte_table_hash_ext_entry_add,
diff --git a/test/test-pipeline/pipeline_hash.c b/test/test-pipeline/pipeline_hash.c
old mode 100644
new mode 100755
index 2f8c625..13d309d
--- a/test/test-pipeline/pipeline_hash.c
+++ b/test/test-pipeline/pipeline_hash.c
@@ -175,15 +175,15 @@ app_main_loop_worker_pipeline_hash(void) {
case e_APP_PIPELINE_HASH_KEY16_EXT:
case e_APP_PIPELINE_HASH_KEY32_EXT:
{
- struct rte_table_hash_ext_params table_hash_params = {
+ struct rte_table_hash_params table_hash_params = {
+ .name = "TABLE",
.key_size = key_size,
+ .key_offset = APP_METADATA_OFFSET(32),
+ .key_mask = NULL,
.n_keys = 1 << 24,
.n_buckets = 1 << 22,
- .n_buckets_ext = 1 << 21,
- .f_hash = test_hash,
+ .f_hash = (rte_table_hash_op_hash)test_hash,
.seed = 0,
- .signature_offset = APP_METADATA_OFFSET(0),
- .key_offset = APP_METADATA_OFFSET(32),
};
struct rte_pipeline_table_params table_params = {
--
2.7.4
More information about the dev
mailing list