[dpdk-dev] [PATCH v1 11/29] net/mlx4: allocate drop flow resources on demand

[Adrien Mazarguil]

Verbs QP and CQ resources for drop flow rules do not need to be permanently
allocated, only when at least one rule needs them.

Besides, struct rte_flow_drop is outside the mlx4 PMD name space and should
never have been defined there. struct rte_flow is currently the only
exception to this rule.

Signed-off-by: Adrien Mazarguil <adrien.mazarguil at 6wind.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro at 6wind.com>
---
 drivers/net/mlx4/mlx4.h      |   3 +-
 drivers/net/mlx4/mlx4_flow.c | 138 ++++++++++++++++++++------------------
 2 files changed, 74 insertions(+), 67 deletions(-)

diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 1799951..f71679b 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -88,6 +88,7 @@ enum {
 /** Driver name reported to lower layers and used in log output. */
 #define MLX4_DRIVER_NAME "net_mlx4"
 
+struct mlx4_drop;
 struct rxq;
 struct txq;
 struct rte_flow;
@@ -108,7 +109,7 @@ struct priv {
 	uint32_t intr_alarm:1; /**< An interrupt alarm is scheduled. */
 	uint32_t isolated:1; /**< Toggle isolated mode. */
 	struct rte_intr_handle intr_handle; /**< Port interrupt handle. */
-	struct rte_flow_drop *flow_drop_queue; /**< Flow drop queue. */
+	struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */
 	LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */
 };
 
diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c
index 0a736b1..658c92f 100644
--- a/drivers/net/mlx4/mlx4_flow.c
+++ b/drivers/net/mlx4/mlx4_flow.c
@@ -125,9 +125,12 @@ struct mlx4_flow_proc_item {
 	const enum rte_flow_item_type *const next_item;
 };
 
-struct rte_flow_drop {
-	struct ibv_qp *qp; /**< Verbs queue pair. */
-	struct ibv_cq *cq; /**< Verbs completion queue. */
+/** Shared resources for drop flow rules. */
+struct mlx4_drop {
+	struct ibv_qp *qp; /**< QP target. */
+	struct ibv_cq *cq; /**< CQ associated with above QP. */
+	struct priv *priv; /**< Back pointer to private data. */
+	uint32_t refcnt; /**< Reference count. */
 };
 
 /**
@@ -744,76 +747,73 @@ mlx4_flow_validate(struct rte_eth_dev *dev,
 }
 
 /**
- * Destroy a drop queue.
+ * Get a drop flow rule resources instance.
  *
  * @param priv
  *   Pointer to private structure.
+ *
+ * @return
+ *   Pointer to drop flow resources on success, NULL otherwise and rte_errno
+ *   is set.
  */
-static void
-mlx4_flow_destroy_drop_queue(struct priv *priv)
+static struct mlx4_drop *
+mlx4_drop_get(struct priv *priv)
 {
-	if (priv->flow_drop_queue) {
-		struct rte_flow_drop *fdq = priv->flow_drop_queue;
+	struct mlx4_drop *drop = priv->drop;
 
-		priv->flow_drop_queue = NULL;
-		claim_zero(ibv_destroy_qp(fdq->qp));
-		claim_zero(ibv_destroy_cq(fdq->cq));
-		rte_free(fdq);
+	if (drop) {
+		assert(drop->refcnt);
+		assert(drop->priv == priv);
+		++drop->refcnt;
+		return drop;
 	}
+	drop = rte_malloc(__func__, sizeof(*drop), 0);
+	if (!drop)
+		goto error;
+	*drop = (struct mlx4_drop){
+		.priv = priv,
+		.refcnt = 1,
+	};
+	drop->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
+	if (!drop->cq)
+		goto error;
+	drop->qp = ibv_create_qp(priv->pd,
+				 &(struct ibv_qp_init_attr){
+					.send_cq = drop->cq,
+					.recv_cq = drop->cq,
+					.qp_type = IBV_QPT_RAW_PACKET,
+				 });
+	if (!drop->qp)
+		goto error;
+	priv->drop = drop;
+	return drop;
+error:
+	if (drop->qp)
+		claim_zero(ibv_destroy_qp(drop->qp));
+	if (drop->cq)
+		claim_zero(ibv_destroy_cq(drop->cq));
+	if (drop)
+		rte_free(drop);
+	rte_errno = ENOMEM;
+	return NULL;
 }
 
 /**
- * Create a single drop queue for all drop flows.
+ * Give back a drop flow rule resources instance.
  *
- * @param priv
- *   Pointer to private structure.
- *
- * @return
- *   0 on success, negative value otherwise.
+ * @param drop
+ *   Pointer to drop flow rule resources.
  */
-static int
-mlx4_flow_create_drop_queue(struct priv *priv)
+static void
+mlx4_drop_put(struct mlx4_drop *drop)
 {
-	struct ibv_qp *qp;
-	struct ibv_cq *cq;
-	struct rte_flow_drop *fdq;
-
-	fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
-	if (!fdq) {
-		ERROR("Cannot allocate memory for drop struct");
-		goto err;
-	}
-	cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
-	if (!cq) {
-		ERROR("Cannot create drop CQ");
-		goto err_create_cq;
-	}
-	qp = ibv_create_qp(priv->pd,
-			   &(struct ibv_qp_init_attr){
-				.send_cq = cq,
-				.recv_cq = cq,
-				.cap = {
-					.max_recv_wr = 1,
-					.max_recv_sge = 1,
-				},
-				.qp_type = IBV_QPT_RAW_PACKET,
-			   });
-	if (!qp) {
-		ERROR("Cannot create drop QP");
-		goto err_create_qp;
-	}
-	*fdq = (struct rte_flow_drop){
-		.qp = qp,
-		.cq = cq,
-	};
-	priv->flow_drop_queue = fdq;
-	return 0;
-err_create_qp:
-	claim_zero(ibv_destroy_cq(cq));
-err_create_cq:
-	rte_free(fdq);
-err:
-	return -1;
+	assert(drop->refcnt);
+	if (--drop->refcnt)
+		return;
+	drop->priv->drop = NULL;
+	claim_zero(ibv_destroy_qp(drop->qp));
+	claim_zero(ibv_destroy_cq(drop->cq));
+	rte_free(drop);
 }
 
 /**
@@ -846,6 +846,8 @@ mlx4_flow_toggle(struct priv *priv,
 			return 0;
 		claim_zero(ibv_destroy_flow(flow->ibv_flow));
 		flow->ibv_flow = NULL;
+		if (flow->drop)
+			mlx4_drop_put(priv->drop);
 		return 0;
 	}
 	if (flow->ibv_flow)
@@ -864,14 +866,21 @@ mlx4_flow_toggle(struct priv *priv,
 		qp = rxq->qp;
 	}
 	if (flow->drop) {
-		assert(priv->flow_drop_queue);
-		qp = priv->flow_drop_queue->qp;
+		mlx4_drop_get(priv);
+		if (!priv->drop) {
+			err = rte_errno;
+			msg = "resources for drop flow rule cannot be created";
+			goto error;
+		}
+		qp = priv->drop->qp;
 	}
 	assert(qp);
 	assert(flow->ibv_attr);
 	flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
 	if (flow->ibv_flow)
 		return 0;
+	if (flow->drop)
+		mlx4_drop_put(priv->drop);
 	err = errno;
 	msg = "flow rule rejected by device";
 error:
@@ -995,7 +1004,7 @@ mlx4_flow_stop(struct priv *priv)
 	     flow = LIST_NEXT(flow, next)) {
 		claim_zero(mlx4_flow_toggle(priv, flow, 0, NULL));
 	}
-	mlx4_flow_destroy_drop_queue(priv);
+	assert(!priv->drop);
 }
 
 /**
@@ -1013,9 +1022,6 @@ mlx4_flow_start(struct priv *priv)
 	int ret;
 	struct rte_flow *flow;
 
-	ret = mlx4_flow_create_drop_queue(priv);
-	if (ret)
-		return -1;
 	for (flow = LIST_FIRST(&priv->flows);
 	     flow;
 	     flow = LIST_NEXT(flow, next)) {
-- 
2.1.4