[dpdk-dev] [PATCH 4/6] net/mlx5: synchronize flow counter pool creation
Suanming Mou
suanmingm at nvidia.com
Tue Oct 6 13:38:51 CEST 2020
Currently, counter operations are not thread safe as the counter
pools' array resize is not protected.
This commit protects the container pools' array resize using a spinlock.
The original counter pool statistic memory allocate is moved to the
host thread in order to minimize the critical section. Since that pool
statistic memory is required only in query time. The container pools'
array should be resized by the user threads, the new pool may be used
by other rte_flow APIs before the host thread resize is done, if the
pool is not saved to the pools' array, the specified counter memory will
not be found as the pool is not saved to the counter management pool
array. The pool raw statistic memory will be filled in host thread.
The shared counters will be protected in other commit.
Signed-off-by: Suanming Mou <suanmingm at nvidia.com>
---
drivers/net/mlx5/mlx5.c | 12 ++-
drivers/net/mlx5/mlx5.h | 11 ++-
drivers/net/mlx5/mlx5_flow.c | 127 ++++++++++++++++++++++++++++++--
drivers/net/mlx5/mlx5_flow_dv.c | 146 ++++++-------------------------------
drivers/net/mlx5/mlx5_flow_verbs.c | 5 +-
5 files changed, 160 insertions(+), 141 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 96cebba..79c5563 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -465,8 +465,7 @@ struct mlx5_flow_id_pool *
sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET;
sh->cmng.max_id = -1;
sh->cmng.last_pool_idx = POOL_IDX_INVALID;
- TAILQ_INIT(&sh->cmng.pool_list);
- rte_spinlock_init(&sh->cmng.resize_sl);
+ rte_spinlock_init(&sh->cmng.pool_update_sl);
for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++)
TAILQ_INIT(&sh->cmng.counters[i]);
rte_spinlock_init(&sh->cmng.csl);
@@ -499,7 +498,7 @@ struct mlx5_flow_id_pool *
mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh)
{
struct mlx5_counter_stats_mem_mng *mng;
- int j;
+ int i, j;
int retries = 1024;
rte_errno = 0;
@@ -512,9 +511,10 @@ struct mlx5_flow_id_pool *
if (sh->cmng.pools) {
struct mlx5_flow_counter_pool *pool;
+ int n_valid = sh->cmng.n_valid;
- pool = TAILQ_FIRST(&sh->cmng.pool_list);
- while (pool) {
+ for (i = 0; i < n_valid; ++i) {
+ pool = sh->cmng.pools[i];
if (!IS_EXT_POOL(pool) && pool->min_dcs)
claim_zero(mlx5_devx_cmd_destroy
(pool->min_dcs));
@@ -530,9 +530,7 @@ struct mlx5_flow_id_pool *
(MLX5_GET_POOL_CNT_EXT
(pool, j)->dcs));
}
- TAILQ_REMOVE(&sh->cmng.pool_list, pool, next);
mlx5_free(pool);
- pool = TAILQ_FIRST(&sh->cmng.pool_list);
}
mlx5_free(sh->cmng.pools);
}
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index a3d4ad9..8c951e2 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -395,7 +395,11 @@ struct mlx5_flow_counter_pool {
volatile uint32_t query_gen:1; /* Query round. */
rte_spinlock_t sl; /* The pool lock. */
struct mlx5_counter_stats_raw *raw;
- struct mlx5_counter_stats_raw *raw_hw; /* The raw on HW working. */
+ union {
+ struct rte_eth_dev *dev; /* The counter pool create device. */
+ struct mlx5_counter_stats_raw *raw_hw;
+ /* The raw on HW working. */
+ };
};
struct mlx5_counter_stats_raw;
@@ -419,16 +423,15 @@ struct mlx5_counter_stats_raw {
/* Counter global management structure. */
struct mlx5_flow_counter_mng {
- rte_atomic16_t n_valid; /* Number of valid pools. */
+ volatile uint16_t n_valid; /* Number of valid pools. */
uint16_t n; /* Number of pools. */
uint16_t last_pool_idx; /* Last used pool index */
int min_id; /* The minimum counter ID in the pools. */
int max_id; /* The maximum counter ID in the pools. */
- rte_spinlock_t resize_sl; /* The resize lock. */
+ rte_spinlock_t pool_update_sl; /* The pool update lock. */
rte_spinlock_t csl; /* The counter free list lock. */
struct mlx5_counters counters[MLX5_COUNTER_TYPE_MAX];
/* Free counter list. */
- struct mlx5_counter_pools pool_list; /* Counter pool list. */
struct mlx5_flow_counter_pool **pools; /* Counter pool array. */
struct mlx5_counter_stats_mem_mng *mem_mng;
/* Hold the memory management for the next allocated pools raws. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c280f56..a9664b8 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -13,6 +13,7 @@
#include <rte_common.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
+#include <rte_eal_paging.h>
#include <rte_flow.h>
#include <rte_cycles.h>
#include <rte_flow_driver.h>
@@ -29,6 +30,7 @@
#include "mlx5_flow.h"
#include "mlx5_flow_os.h"
#include "mlx5_rxtx.h"
+#include "mlx5_common_os.h"
/** Device flow drivers. */
extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
@@ -5880,6 +5882,116 @@ struct mlx5_meter_domains_infos *
return -ENOTSUP;
}
+/**
+ * Allocate a new memory for the counter values wrapped by all the needed
+ * management.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in] raws_n
+ * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise.
+ */
+static int
+mlx5_flow_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_dev_ctx_shared *sh = priv->sh;
+ struct mlx5_devx_mkey_attr mkey_attr;
+ struct mlx5_counter_stats_mem_mng *mem_mng;
+ volatile struct flow_counter_stats *raw_data;
+ int size = (sizeof(struct flow_counter_stats) *
+ MLX5_COUNTERS_PER_POOL +
+ sizeof(struct mlx5_counter_stats_raw)) * raws_n +
+ sizeof(struct mlx5_counter_stats_mem_mng);
+ size_t pgsize = rte_mem_page_size();
+ if (pgsize == (size_t)-1) {
+ DRV_LOG(ERR, "Failed to get mem page size");
+ rte_errno = ENOMEM;
+ return -ENOMEM;
+ }
+ uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
+ SOCKET_ID_ANY);
+ int i;
+
+ if (!mem) {
+ rte_errno = ENOMEM;
+ return -ENOMEM;
+ }
+ mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
+ size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
+ mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!mem_mng->umem) {
+ rte_errno = errno;
+ mlx5_free(mem);
+ return -rte_errno;
+ }
+ mkey_attr.addr = (uintptr_t)mem;
+ mkey_attr.size = size;
+ mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
+ mkey_attr.pd = sh->pdn;
+ mkey_attr.log_entity_size = 0;
+ mkey_attr.pg_access = 0;
+ mkey_attr.klm_array = NULL;
+ mkey_attr.klm_num = 0;
+ if (priv->config.hca_attr.relaxed_ordering_write &&
+ priv->config.hca_attr.relaxed_ordering_read &&
+ !haswell_broadwell_cpu)
+ mkey_attr.relaxed_ordering = 1;
+ mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
+ if (!mem_mng->dm) {
+ mlx5_glue->devx_umem_dereg(mem_mng->umem);
+ rte_errno = errno;
+ mlx5_free(mem);
+ return -rte_errno;
+ }
+ mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
+ raw_data = (volatile struct flow_counter_stats *)mem;
+ for (i = 0; i < raws_n; ++i) {
+ mem_mng->raws[i].mem_mng = mem_mng;
+ mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
+ }
+ for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
+ LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
+ mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
+ next);
+ LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
+ priv->sh->cmng.mem_mng = mem_mng;
+ return 0;
+}
+
+/**
+ * Set the statistic memory to the new counter pool.
+ *
+ * @param[in] cmng
+ * Pointer to the counter management.
+ * @param[in] pool
+ * Pointer to the pool to set the statistic memory.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise.
+ */
+static int
+mlx5_flow_set_counter_stat_mem(struct mlx5_flow_counter_mng *cmng,
+ struct mlx5_flow_counter_pool *pool)
+{
+ /* Resize statistic memory once used out. */
+ if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
+ mlx5_flow_create_counter_stat_mem_mng(pool->dev,
+ MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES)) {
+ DRV_LOG(ERR, "Cannot resize counter stat mem.");
+ return -1;
+ }
+ MLX5_ASSERT(pool->index < n_valid);
+ pool->raw = cmng->mem_mng->raws + pool->index %
+ MLX5_CNT_CONTAINER_RESIZE;
+ pool->raw_hw = NULL;
+ return 0;
+}
+
#define MLX5_POOL_QUERY_FREQ_US 1000000
/**
@@ -5894,7 +6006,7 @@ struct mlx5_meter_domains_infos *
{
uint32_t pools_n, us;
- pools_n = rte_atomic16_read(&sh->cmng.n_valid);
+ pools_n = sh->cmng.n_valid;
us = MLX5_POOL_QUERY_FREQ_US / pools_n;
DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
@@ -5920,16 +6032,21 @@ struct mlx5_meter_domains_infos *
uint16_t pool_index = sh->cmng.pool_index;
struct mlx5_flow_counter_mng *cmng = &sh->cmng;
struct mlx5_flow_counter_pool *pool;
+ int n_valid;
if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
goto set_alarm;
- rte_spinlock_lock(&cmng->resize_sl);
+ rte_spinlock_lock(&cmng->pool_update_sl);
if (!cmng->pools) {
- rte_spinlock_unlock(&cmng->resize_sl);
+ rte_spinlock_unlock(&cmng->pool_update_sl);
goto set_alarm;
}
pool = cmng->pools[pool_index];
- rte_spinlock_unlock(&cmng->resize_sl);
+ n_valid = cmng->n_valid;
+ rte_spinlock_unlock(&cmng->pool_update_sl);
+ /* Set the statistic memory to the new created pool. */
+ if ((!pool->raw && mlx5_flow_set_counter_stat_mem(cmng, pool)))
+ goto set_alarm;
if (pool->raw_hw)
/* There is a pool query in progress. */
goto set_alarm;
@@ -5962,7 +6079,7 @@ struct mlx5_meter_domains_infos *
LIST_REMOVE(pool->raw_hw, next);
sh->cmng.pending_queries++;
pool_index++;
- if (pool_index >= rte_atomic16_read(&cmng->n_valid))
+ if (pool_index >= n_valid)
pool_index = 0;
set_alarm:
sh->cmng.pool_index = pool_index;
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 43d8ea8..31d7fe4 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -4231,7 +4231,7 @@ struct field_modify_info modify_tcp[] = {
* ID is sequence increasing, and the last pool should be the needed
* one.
*/
- i = rte_atomic16_read(&cmng->n_valid);
+ i = cmng->n_valid;
while (i--) {
struct mlx5_flow_counter_pool *pool = cmng->pools[i];
@@ -4242,83 +4242,6 @@ struct field_modify_info modify_tcp[] = {
}
/**
- * Allocate a new memory for the counter values wrapped by all the needed
- * management.
- *
- * @param[in] dev
- * Pointer to the Ethernet device structure.
- * @param[in] raws_n
- * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
- *
- * @return
- * The new memory management pointer on success, otherwise NULL and rte_errno
- * is set.
- */
-static struct mlx5_counter_stats_mem_mng *
-flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_dev_ctx_shared *sh = priv->sh;
- struct mlx5_devx_mkey_attr mkey_attr;
- struct mlx5_counter_stats_mem_mng *mem_mng;
- volatile struct flow_counter_stats *raw_data;
- int size = (sizeof(struct flow_counter_stats) *
- MLX5_COUNTERS_PER_POOL +
- sizeof(struct mlx5_counter_stats_raw)) * raws_n +
- sizeof(struct mlx5_counter_stats_mem_mng);
- size_t pgsize = rte_mem_page_size();
- if (pgsize == (size_t)-1) {
- DRV_LOG(ERR, "Failed to get mem page size");
- rte_errno = ENOMEM;
- return NULL;
- }
- uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
- SOCKET_ID_ANY);
- int i;
-
- if (!mem) {
- rte_errno = ENOMEM;
- return NULL;
- }
- mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
- size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
- mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
- IBV_ACCESS_LOCAL_WRITE);
- if (!mem_mng->umem) {
- rte_errno = errno;
- mlx5_free(mem);
- return NULL;
- }
- mkey_attr.addr = (uintptr_t)mem;
- mkey_attr.size = size;
- mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
- mkey_attr.pd = sh->pdn;
- mkey_attr.log_entity_size = 0;
- mkey_attr.pg_access = 0;
- mkey_attr.klm_array = NULL;
- mkey_attr.klm_num = 0;
- if (priv->config.hca_attr.relaxed_ordering_write &&
- priv->config.hca_attr.relaxed_ordering_read &&
- !haswell_broadwell_cpu)
- mkey_attr.relaxed_ordering = 1;
- mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
- if (!mem_mng->dm) {
- mlx5_glue->devx_umem_dereg(mem_mng->umem);
- rte_errno = errno;
- mlx5_free(mem);
- return NULL;
- }
- mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
- raw_data = (volatile struct flow_counter_stats *)mem;
- for (i = 0; i < raws_n; ++i) {
- mem_mng->raws[i].mem_mng = mem_mng;
- mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
- }
- LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
- return mem_mng;
-}
-
-/**
* Resize a counter container.
*
* @param[in] dev
@@ -4332,7 +4255,6 @@ struct field_modify_info modify_tcp[] = {
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
- struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
void *old_pools = cmng->pools;
uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
@@ -4345,30 +4267,8 @@ struct field_modify_info modify_tcp[] = {
if (old_pools)
memcpy(pools, old_pools, cmng->n *
sizeof(struct mlx5_flow_counter_pool *));
- /*
- * Fallback mode query the counter directly, no background query
- * resources are needed.
- */
- if (!priv->counter_fallback) {
- int i;
-
- mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
- MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
- if (!mem_mng) {
- mlx5_free(pools);
- return -ENOMEM;
- }
- for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
- LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
- mem_mng->raws +
- MLX5_CNT_CONTAINER_RESIZE +
- i, next);
- }
- rte_spinlock_lock(&cmng->resize_sl);
cmng->n = resize;
- cmng->mem_mng = mem_mng;
cmng->pools = pools;
- rte_spinlock_unlock(&cmng->resize_sl);
if (old_pools)
mlx5_free(old_pools);
return 0;
@@ -4406,11 +4306,15 @@ struct field_modify_info modify_tcp[] = {
return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
0, pkts, bytes, 0, NULL, NULL, 0);
}
-
rte_spinlock_lock(&pool->sl);
- offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
- *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
- *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
+ if (!pool->raw) {
+ *pkts = 0;
+ *bytes = 0;
+ } else {
+ offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
+ *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
+ *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
+ }
rte_spinlock_unlock(&pool->sl);
return 0;
}
@@ -4437,12 +4341,9 @@ struct field_modify_info modify_tcp[] = {
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_counter_pool *pool;
struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
- int16_t n_valid = rte_atomic16_read(&cmng->n_valid);
uint32_t fallback = priv->counter_fallback;
uint32_t size = sizeof(*pool);
- if (cmng->n == n_valid && flow_dv_container_resize(dev))
- return NULL;
size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
size += (!fallback ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
@@ -4451,23 +4352,25 @@ struct field_modify_info modify_tcp[] = {
rte_errno = ENOMEM;
return NULL;
}
- if (!fallback) {
- pool->min_dcs = dcs;
- pool->raw = cmng->mem_mng->raws + n_valid %
- MLX5_CNT_CONTAINER_RESIZE;
- }
- pool->raw_hw = NULL;
+ pool->raw = NULL;
pool->type = 0;
- pool->type |= (!fallback ? 0 : CNT_POOL_TYPE_EXT);
pool->type |= (!age ? 0 : CNT_POOL_TYPE_AGE);
pool->query_gen = 0;
+ pool->min_dcs = dcs;
rte_spinlock_init(&pool->sl);
TAILQ_INIT(&pool->counters[0]);
TAILQ_INIT(&pool->counters[1]);
- TAILQ_INSERT_HEAD(&cmng->pool_list, pool, next);
- pool->index = n_valid;
- cmng->pools[n_valid] = pool;
- if (fallback) {
+ rte_spinlock_lock(&cmng->pool_update_sl);
+ pool->index = cmng->n_valid;
+ if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
+ mlx5_free(pool);
+ rte_spinlock_unlock(&cmng->pool_update_sl);
+ return NULL;
+ }
+ cmng->pools[pool->index] = pool;
+ pool->dev = dev;
+ cmng->n_valid++;
+ if (unlikely(fallback)) {
int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
if (base < cmng->min_id)
@@ -4475,10 +4378,9 @@ struct field_modify_info modify_tcp[] = {
if (base > cmng->max_id)
cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
cmng->last_pool_idx = pool->index;
+ pool->type |= CNT_POOL_TYPE_EXT;
}
- /* Pool initialization must be updated before host thread access. */
- rte_io_wmb();
- rte_atomic16_add(&cmng->n_valid, 1);
+ rte_spinlock_unlock(&cmng->pool_update_sl);
return pool;
}
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index f3b0e89..69ecc27 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -259,7 +259,7 @@
struct mlx5_flow_counter_ext *cnt_ext = NULL;
struct mlx5_flow_counter *cnt = NULL;
union mlx5_l3t_data data;
- uint32_t n_valid = rte_atomic16_read(&cmng->n_valid);
+ uint32_t n_valid = cmng->n_valid;
uint32_t pool_idx, cnt_idx;
uint32_t i;
int ret;
@@ -317,8 +317,7 @@
cnt = MLX5_POOL_GET_CNT(pool, 0);
cmng->pools[n_valid] = pool;
pool_idx = n_valid;
- rte_atomic16_add(&cmng->n_valid, 1);
- TAILQ_INSERT_HEAD(&cmng->pool_list, pool, next);
+ cmng->n_valid++;
}
i = MLX5_CNT_ARRAY_IDX(pool, cnt);
cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i);
--
1.8.3.1
More information about the dev
mailing list