[dpdk-dev] [PATCH 2/2] net/mlx5: optimize the hash list entry memory

Suanming Mou suanmingm at nvidia.com
Wed Dec 2 12:39:25 CET 2020


Currently, the hash list saves the hash key in the hash entry. And the
key is mostly used to get the bucket index only.

Save the entire 64 bits key to the entry will not be a good option if
the key is only used to get the bucket index. Since 64 bits costs more
memory for the entry, mostly the signature data in the key only uses
32 bits. And in the unregister function, the key in the entry causes
extra bucket index calculation.

This commit saves the bucket index to the entry instead of the hash key.
For the hash list like table, tag and mreg_copy which save the signature
data in the key, the signature data is moved to the resource data struct
itself.

Signed-off-by: Suanming Mou <suanmingm at nvidia.com>
Acked-by: Matan Azrad <matan at nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c |  5 ++--
 drivers/net/mlx5/mlx5.c          |  3 ++-
 drivers/net/mlx5/mlx5_flow.c     | 43 ++++++++++++++++++++++++------
 drivers/net/mlx5/mlx5_flow.h     | 23 +++++++++++++---
 drivers/net/mlx5/mlx5_flow_dv.c  | 57 +++++++++++++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.c    | 21 +++------------
 drivers/net/mlx5/mlx5_utils.h    |  4 +--
 7 files changed, 107 insertions(+), 49 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 4c863db..6c501c0 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -264,7 +264,8 @@
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
 					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb, NULL,
+					  flow_dv_tag_create_cb,
+					  flow_dv_tag_match_cb,
 					  flow_dv_tag_remove_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
@@ -1564,7 +1565,7 @@
 						      MLX5_FLOW_MREG_HTABLE_SZ,
 						      0, 0,
 						      flow_dv_mreg_create_cb,
-						      NULL,
+						      flow_dv_mreg_match_cb,
 						      flow_dv_mreg_remove_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index ca3667a..7d3f18c 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1152,7 +1152,8 @@ struct mlx5_dev_ctx_shared *
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb, NULL,
+					  0, 0, flow_dv_tbl_create_cb,
+					  flow_dv_tbl_match_cb,
 					  flow_dv_tbl_remove_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 52ade39..82e24d7 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3574,6 +3574,17 @@ struct mlx5_translated_shared_action {
 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 		  uint32_t flow_idx);
 
+int
+flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
+		      struct mlx5_hlist_entry *entry,
+		      uint64_t key, void *cb_ctx __rte_unused)
+{
+	struct mlx5_flow_mreg_copy_resource *mcp_res =
+		container_of(entry, typeof(*mcp_res), hlist_ent);
+
+	return mcp_res->mark_id != key;
+}
+
 struct mlx5_hlist_entry *
 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 		       void *cb_ctx)
@@ -3675,6 +3686,7 @@ struct mlx5_hlist_entry *
 		return NULL;
 	}
 	mcp_res->idx = idx;
+	mcp_res->mark_id = mark_id;
 	/*
 	 * The copy Flows are not included in any list. There
 	 * ones are referenced from other Flows and can not
@@ -5022,7 +5034,6 @@ struct mlx5_hlist_entry *
 	struct mlx5_flow_dv_sample_resource *sample_res;
 	struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
 	struct mlx5_flow_tbl_resource *sfx_tbl;
-	union mlx5_flow_tbl_key sfx_table_key;
 #endif
 	size_t act_size;
 	size_t item_size;
@@ -5082,10 +5093,9 @@ struct mlx5_hlist_entry *
 					sample_res->normal_path_tbl;
 		sfx_tbl_data = container_of(sfx_tbl,
 					struct mlx5_flow_tbl_data_entry, tbl);
-		sfx_table_key.v64 = sfx_tbl_data->entry.key;
 		sfx_attr.group = sfx_attr.transfer ?
-					(sfx_table_key.table_id - 1) :
-					 sfx_table_key.table_id;
+					(sfx_tbl_data->table_id - 1) :
+					 sfx_tbl_data->table_id;
 		flow_split_info->prefix_layers =
 				flow_get_prefix_layer_flags(dev_flow);
 		flow_split_info->prefix_mark = dev_flow->handle->mark;
@@ -7353,13 +7363,28 @@ int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
 	mlx5_free(tte);
 }
 
+static int
+mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
+				  struct mlx5_hlist_entry *entry,
+				  uint64_t key, void *cb_ctx __rte_unused)
+{
+	union tunnel_tbl_key tbl = {
+		.val = key,
+	};
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
+}
+
 static struct mlx5_hlist_entry *
-mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list,
-				   uint64_t key __rte_unused,
+mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
 				   void *ctx __rte_unused)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct tunnel_tbl_entry *tte;
+	union tunnel_tbl_key tbl = {
+		.val = key,
+	};
 
 	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
 			  sizeof(*tte), 0,
@@ -7378,6 +7403,8 @@ int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
 		goto err;
 	}
 	tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
+	tte->tunnel_id = tbl.tunnel_id;
+	tte->group = tbl.group;
 	return &tte->hash;
 err:
 	if (tte)
@@ -7517,7 +7544,7 @@ struct tunnel_db_find_tunnel_id_ctx {
 	}
 	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
-					   NULL,
+					   mlx5_flow_tunnel_grp2tbl_match_cb,
 					   mlx5_flow_tunnel_grp2tbl_remove_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(ipool, id);
@@ -7623,7 +7650,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
 					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
-					 NULL,
+					 mlx5_flow_tunnel_grp2tbl_match_cb,
 					 mlx5_flow_tunnel_grp2tbl_remove_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index a249c29..d85dd19 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -437,6 +437,7 @@ struct mlx5_flow_dv_tag_resource {
 	/**< Tag action object. */
 	uint32_t refcnt; /**< Reference counter. */
 	uint32_t idx; /**< Index for the index memory pool. */
+	uint32_t tag_id; /**< Tag ID. */
 };
 
 /*
@@ -509,6 +510,7 @@ struct mlx5_flow_mreg_copy_resource {
 	/* List entry for device flows. */
 	uint32_t idx;
 	uint32_t rix_flow; /* Built flow for copy. */
+	uint32_t mark_id;
 };
 
 /* Table tunnel parameter. */
@@ -532,9 +534,13 @@ struct mlx5_flow_tbl_data_entry {
 	/**< tunnel offload */
 	const struct mlx5_flow_tunnel *tunnel;
 	uint32_t group_id;
-	bool external;
-	bool tunnel_offload; /* Tunnel offlod table or not. */
-	bool is_egress; /**< Egress table. */
+	uint32_t external:1;
+	uint32_t tunnel_offload:1; /* Tunnel offlod table or not. */
+	uint32_t is_egress:1; /**< Egress table. */
+	uint32_t is_transfer:1; /**< Transfer table. */
+	uint32_t dummy:1; /**<  DR table. */
+	uint32_t reserve:27; /**< Reserved to future using. */
+	uint32_t table_id; /**< Table ID. */
 };
 
 /* Sub rdma-core actions list. */
@@ -954,6 +960,8 @@ struct mlx5_flow_tunnel_hub {
 struct tunnel_tbl_entry {
 	struct mlx5_hlist_entry hash;
 	uint32_t flow_table;
+	uint32_t tunnel_id;
+	uint32_t group;
 };
 
 static inline uint32_t
@@ -1416,6 +1424,9 @@ int mlx5_flow_meter_flush(struct rte_eth_dev *dev,
 /* Hash list callbacks for flow tables: */
 struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
 					       uint64_t key, void *entry_ctx);
+int flow_dv_tbl_match_cb(struct mlx5_hlist *list,
+			 struct mlx5_hlist_entry *entry, uint64_t key,
+			 void *cb_ctx);
 void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			   struct mlx5_hlist_entry *entry);
 struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
@@ -1425,6 +1436,9 @@ struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 
 struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
 					       uint64_t key, void *cb_ctx);
+int flow_dv_tag_match_cb(struct mlx5_hlist *list,
+			 struct mlx5_hlist_entry *entry, uint64_t key,
+			 void *cb_ctx);
 void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
 			   struct mlx5_hlist_entry *entry);
 
@@ -1438,6 +1452,9 @@ void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
 
 struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
 						uint64_t key, void *ctx);
+int flow_dv_mreg_match_cb(struct mlx5_hlist *list,
+			  struct mlx5_hlist_entry *entry, uint64_t key,
+			  void *cb_ctx);
 void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
 			    struct mlx5_hlist_entry *entry);
 
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index aa21ff9..30c7441 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -2780,8 +2780,7 @@ struct field_modify_info modify_tcp[] = {
 	cache_resource = container_of(entry,
 				      struct mlx5_flow_dv_encap_decap_resource,
 				      entry);
-	if (resource->entry.key == cache_resource->entry.key &&
-	    resource->reformat_type == cache_resource->reformat_type &&
+	if (resource->reformat_type == cache_resource->reformat_type &&
 	    resource->ft_type == cache_resource->ft_type &&
 	    resource->flags == cache_resource->flags &&
 	    resource->size == cache_resource->size &&
@@ -2899,18 +2898,16 @@ struct mlx5_hlist_entry *
 		.error = error,
 		.data = resource,
 	};
+	uint64_t key64;
 
 	resource->flags = dev_flow->dv.group ? 0 : 1;
-	resource->entry.key =  __rte_raw_cksum(&encap_decap_key.v32,
-					       sizeof(encap_decap_key.v32), 0);
+	key64 =  __rte_raw_cksum(&encap_decap_key.v32,
+				 sizeof(encap_decap_key.v32), 0);
 	if (resource->reformat_type !=
 	    MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
 	    resource->size)
-		resource->entry.key = __rte_raw_cksum(resource->buf,
-						      resource->size,
-						      resource->entry.key);
-	entry = mlx5_hlist_register(sh->encaps_decaps, resource->entry.key,
-				    &ctx);
+		key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
+	entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -4534,6 +4531,7 @@ struct mlx5_hlist_entry *
 		.error = error,
 		.data = resource,
 	};
+	uint64_t key64;
 
 	resource->flags = dev_flow->dv.group ? 0 :
 			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
@@ -4542,8 +4540,8 @@ struct mlx5_hlist_entry *
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
-	resource->entry.key = __rte_raw_cksum(&resource->ft_type, key_len, 0);
-	entry = mlx5_hlist_register(sh->modify_cmds, resource->entry.key, &ctx);
+	key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
+	entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -7989,9 +7987,12 @@ struct mlx5_hlist_entry *
 	tbl_data->idx = idx;
 	tbl_data->tunnel = tt_prm->tunnel;
 	tbl_data->group_id = tt_prm->group_id;
-	tbl_data->external = tt_prm->external;
+	tbl_data->external = !!tt_prm->external;
 	tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
 	tbl_data->is_egress = !!key.direction;
+	tbl_data->is_transfer = !!key.domain;
+	tbl_data->dummy = !!key.dummy;
+	tbl_data->table_id = key.table_id;
 	tbl = &tbl_data->tbl;
 	if (key.dummy)
 		return &tbl_data->entry;
@@ -8032,6 +8033,21 @@ struct mlx5_hlist_entry *
 	return &tbl_data->entry;
 }
 
+int
+flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
+		     struct mlx5_hlist_entry *entry, uint64_t key64,
+		     void *cb_ctx __rte_unused)
+{
+	struct mlx5_flow_tbl_data_entry *tbl_data =
+		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+	union mlx5_flow_tbl_key key = { .v64 = key64 };
+
+	return tbl_data->table_id != key.table_id ||
+	       tbl_data->dummy != key.dummy ||
+	       tbl_data->is_transfer != key.domain ||
+	       tbl_data->is_egress != key.direction;
+}
+
 /**
  * Get a flow table.
  *
@@ -8117,10 +8133,7 @@ struct mlx5_flow_tbl_resource *
 					tbl_data->tunnel->tunnel_id : 0,
 			.group = tbl_data->group_id
 		};
-		union mlx5_flow_tbl_key table_key = {
-			.v64 = entry->key
-		};
-		uint32_t table_id = table_key.table_id;
+		uint32_t table_id = tbl_data->table_id;
 
 		tunnel_grp_hash = tbl_data->tunnel ?
 					tbl_data->tunnel->groups :
@@ -8295,6 +8308,7 @@ struct mlx5_hlist_entry *
 		return NULL;
 	}
 	entry->idx = idx;
+	entry->tag_id = key;
 	ret = mlx5_flow_os_create_flow_action_tag(key,
 						  &entry->action);
 	if (ret) {
@@ -8307,6 +8321,17 @@ struct mlx5_hlist_entry *
 	return &entry->entry;
 }
 
+int
+flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
+		     struct mlx5_hlist_entry *entry, uint64_t key,
+		     void *cb_ctx __rte_unused)
+{
+	struct mlx5_flow_dv_tag_resource *tag =
+		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+
+	return key != tag->tag_id;
+}
+
 /**
  * Find existing tag resource or create and register a new one.
  *
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 42e5d80..07373bf 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -25,14 +25,6 @@
 	mlx5_free(entry);
 }
 
-static int
-mlx5_hlist_default_match_cb(struct mlx5_hlist *h __rte_unused,
-			    struct mlx5_hlist_entry *entry,
-			    uint64_t key, void *ctx __rte_unused)
-{
-	return entry->key != key;
-}
-
 struct mlx5_hlist *
 mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 		  uint32_t flags, mlx5_hlist_create_cb cb_create,
@@ -43,7 +35,7 @@ struct mlx5_hlist *
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || (!cb_create ^ !cb_remove))
+	if (!size || !cb_match || (!cb_create ^ !cb_remove))
 		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
@@ -71,7 +63,7 @@ struct mlx5_hlist *
 	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
 	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
 	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match ? cb_match : mlx5_hlist_default_match_cb;
+	h->cb_match = cb_match;
 	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
 	for (i = 0; i < act_size; i++)
 		rte_rwlock_init(&h->buckets[i].lock);
@@ -166,7 +158,7 @@ struct mlx5_hlist_entry*
 		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
 		goto done;
 	}
-	entry->key = key;
+	entry->idx = idx;
 	entry->ref_cnt = 1;
 	LIST_INSERT_HEAD(first, entry, next);
 	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
@@ -180,12 +172,7 @@ struct mlx5_hlist_entry*
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
 {
-	uint32_t idx;
-
-	if (h->direct_key)
-		idx = (uint32_t)(entry->key & h->mask);
-	else
-		idx = rte_hash_crc_8byte(entry->key, 0) & h->mask;
+	uint32_t idx = entry->idx;
 
 	rte_rwlock_write_lock(&h->buckets[idx].lock);
 	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 6f25768..1971b17 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -274,9 +274,9 @@ struct mlx5_indexed_pool {
  */
 struct mlx5_hlist_entry {
 	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint64_t key; /* user defined 'key', could be the hash signature. */
+	uint32_t idx; /* Bucket index the entry belongs to. */
 	uint32_t ref_cnt; /* Reference count. */
-};
+} __rte_packed;
 
 /** Structure for hash head. */
 LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
-- 
1.8.3.1



More information about the dev mailing list