[dpdk-dev] [PATCH 07/13] net/sfc: factory out libefx-based Tx datapath

Andrew Rybchenko arybchenko at solarflare.com
Thu Mar 2 08:07:13 CET 2017


Split control and datapath to make datapath substitutable and
possibly reusable with alternative control path.

libefx-based Tx datapath is bound to libefx control path, but
other datapaths should be possible to use with alternative
control path(s).

Signed-off-by: Andrew Rybchenko <arybchenko at solarflare.com>
---
 doc/guides/nics/sfc_efx.rst  |   8 ++
 drivers/net/sfc/sfc.h        |   1 +
 drivers/net/sfc/sfc_dp.c     |   4 +-
 drivers/net/sfc/sfc_dp.h     |   1 +
 drivers/net/sfc/sfc_dp_tx.h  | 155 ++++++++++++++++++++++
 drivers/net/sfc/sfc_ethdev.c |  41 +++++-
 drivers/net/sfc/sfc_ev.c     |  50 ++++++--
 drivers/net/sfc/sfc_ev.h     |   8 +-
 drivers/net/sfc/sfc_kvargs.c |   1 +
 drivers/net/sfc/sfc_kvargs.h |   4 +
 drivers/net/sfc/sfc_tso.c    |  22 ++--
 drivers/net/sfc/sfc_tx.c     | 298 ++++++++++++++++++++++++++++++++-----------
 drivers/net/sfc/sfc_tx.h     |  95 +++++++++-----
 13 files changed, 557 insertions(+), 131 deletions(-)
 create mode 100644 drivers/net/sfc/sfc_dp_tx.h

diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst
index 0aa6740..e864ccc 100644
--- a/doc/guides/nics/sfc_efx.rst
+++ b/doc/guides/nics/sfc_efx.rst
@@ -191,6 +191,14 @@ boolean parameters value.
   more efficient than libefx-based and provides richer packet type
   classification, but lacks Rx scatter support.
 
+- ``tx_datapath`` [auto|efx] (default **auto**)
+
+  Choose transmit datapath implementation.
+  **auto** allows the driver itself to make a choice based on firmware
+  features available and required by the datapath implementation.
+  **efx** chooses libefx-based datapath which supports VLAN insertion
+  (full-feature firmware variant only), TSO and multi-segment mbufs.
+
 - ``perf_profile`` [auto|throughput|low-latency] (default **throughput**)
 
   Choose hardware tunning to be optimized for either throughput or
diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
index 2512f2e..893eafe 100644
--- a/drivers/net/sfc/sfc.h
+++ b/drivers/net/sfc/sfc.h
@@ -212,6 +212,7 @@ struct sfc_adapter {
 #endif
 
 	const struct sfc_dp_rx		*dp_rx;
+	const struct sfc_dp_tx		*dp_tx;
 };
 
 /*
diff --git a/drivers/net/sfc/sfc_dp.c b/drivers/net/sfc/sfc_dp.c
index c4d5fb3..47bd500 100644
--- a/drivers/net/sfc/sfc_dp.c
+++ b/drivers/net/sfc/sfc_dp.c
@@ -76,7 +76,9 @@ struct sfc_dp *
 	if (sfc_dp_find_by_name(head, entry->type, entry->name) != NULL) {
 		rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD,
 			"sfc %s dapapath '%s' already registered\n",
-			entry->type == SFC_DP_RX ? "Rx" : "unknown",
+			entry->type == SFC_DP_RX ? "Rx" :
+			entry->type == SFC_DP_TX ? "Tx" :
+			"unknown",
 			entry->name);
 		return EEXIST;
 	}
diff --git a/drivers/net/sfc/sfc_dp.h b/drivers/net/sfc/sfc_dp.h
index 39e1e70..d3e7007 100644
--- a/drivers/net/sfc/sfc_dp.h
+++ b/drivers/net/sfc/sfc_dp.h
@@ -52,6 +52,7 @@
 
 enum sfc_dp_type {
 	SFC_DP_RX = 0,	/**< Receive datapath */
+	SFC_DP_TX,	/**< Transmit datapath */
 };
 
 /** Datapath definition */
diff --git a/drivers/net/sfc/sfc_dp_tx.h b/drivers/net/sfc/sfc_dp_tx.h
new file mode 100644
index 0000000..4879db5
--- /dev/null
+++ b/drivers/net/sfc/sfc_dp_tx.h
@@ -0,0 +1,155 @@
+/*-
+ * Copyright (c) 2016 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SFC_DP_TX_H
+#define _SFC_DP_TX_H
+
+#include <rte_ethdev.h>
+
+#include "sfc_dp.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sfc_dp_txq;
+
+/**
+ * Callback to get control path transmit queue by datapath transmit queue
+ * handle.
+ */
+typedef void * (sfc_dp_txq_get_ctrl_t)(struct sfc_dp_txq *dp_txq);
+
+/** Datapath transmit queue operations */
+struct sfc_dp_txq_ops {
+	sfc_dp_txq_get_ctrl_t		*get_ctrl;
+};
+
+/**
+ * Generic transmit queue information used on data path.
+ * It must be kept as small as it is possible since it is built into
+ * the structure used on datapath.
+ */
+struct sfc_dp_txq {
+	const struct sfc_dp_txq_ops	*ops;
+};
+
+/**
+ * Datapath transmit queue creation arguments.
+ *
+ * The structure is used just to pass information from control path to
+ * datapath. It could be just function arguments, but it would be hardly
+ * readable.
+ */
+struct sfc_dp_tx_qcreate_args {
+	/** Minimum number of unused Tx descriptors to do reap */
+	unsigned int		free_thresh;
+	/** Transmit queue configuration flags */
+	unsigned int		flags;
+	/** Tx queue size */
+	unsigned int		txq_entries;
+};
+
+/**
+ * Allocate and initalize datapath transmit queue.
+ *
+ * @param ctrl		Control path Tx queue opaque handle
+ * @param exception	Datapath exception handler to bail out to control path
+ * @param socket_id	Socket ID to allocate memory
+ * @param args		Tx queue details wrapped in structure
+ * @param dp_txqp	Location for generic datapath transmit queue pointer
+ *
+ * @return 0 or positive errno.
+ */
+typedef int (sfc_dp_tx_qcreate_t)(void *ctrl, sfc_dp_exception_t *exception,
+				  int socket_id,
+				  const struct sfc_dp_tx_qcreate_args *args,
+				  struct sfc_dp_txq **dp_txqp);
+
+/**
+ * Free resources allocated for datapath transmit queue.
+ */
+typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq);
+
+/**
+ * Transmit queue start callback.
+ *
+ * It handovers EvQ to the datapath.
+ */
+typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq,
+				 unsigned int evq_read_ptr,
+				 unsigned int txq_desc_index);
+
+/**
+ * Transmit queue stop function called before the queue flush.
+ *
+ * It returns EvQ to the control path.
+ */
+typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq,
+				 unsigned int *evq_read_ptr);
+
+/**
+ * Transmit queue function called after the queue flush.
+ */
+typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq);
+
+/** Transmit datapath definition */
+struct sfc_dp_tx {
+	struct sfc_dp			dp;
+
+	sfc_dp_tx_qcreate_t		*qcreate;
+	sfc_dp_tx_qdestroy_t		*qdestroy;
+	sfc_dp_tx_qstart_t		*qstart;
+	sfc_dp_tx_qstop_t		*qstop;
+	sfc_dp_tx_qreap_t		*qreap;
+	eth_tx_burst_t			pkt_burst;
+};
+
+static inline struct sfc_dp_tx *
+sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name)
+{
+	struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name);
+
+	return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
+}
+
+static inline struct sfc_dp_tx *
+sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps)
+{
+	struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps);
+
+	return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
+}
+
+extern struct sfc_dp_tx sfc_efx_tx;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_DP_TX_H */
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index f6562f9..c6db730 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -416,7 +416,7 @@
 	if (rc != 0)
 		goto fail_tx_qinit;
 
-	dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq;
+	dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq->dp;
 
 	sfc_adapter_unlock(sa);
 	return 0;
@@ -1232,6 +1232,7 @@
 	struct sfc_adapter *sa = dev->data->dev_private;
 	unsigned int avail_caps = 0;
 	const char *rx_name = NULL;
+	const char *tx_name = NULL;
 	int rc;
 
 	if (sa == NULL || sa->state == SFC_ADAPTER_UNINITIALIZED)
@@ -1279,12 +1280,45 @@
 
 	dev->rx_pkt_burst = sa->dp_rx->pkt_burst;
 
-	dev->tx_pkt_burst = sfc_xmit_pkts;
+	rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH,
+				sfc_kvarg_string_handler, &tx_name);
+	if (rc != 0)
+		goto fail_kvarg_tx_datapath;
+
+	if (tx_name != NULL) {
+		sa->dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name);
+		if (sa->dp_tx == NULL) {
+			sfc_err(sa, "Tx datapath %s not found", tx_name);
+			rc = ENOENT;
+			goto fail_dp_tx;
+		}
+		if (!sfc_dp_match_hw_fw_caps(&sa->dp_tx->dp, avail_caps)) {
+			sfc_err(sa,
+				"Insufficient Hw/FW capabilities to use Tx datapath %s",
+				tx_name);
+			rc = EINVAL;
+			goto fail_dp_tx;
+		}
+	} else {
+		sa->dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps);
+		if (sa->dp_tx == NULL) {
+			sfc_err(sa, "Tx datapath by caps %#x not found",
+				avail_caps);
+			rc = ENOENT;
+			goto fail_dp_tx;
+		}
+	}
+
+	sfc_info(sa, "use %s Tx datapath", sa->dp_tx->dp.name);
+
+	dev->tx_pkt_burst = sa->dp_tx->pkt_burst;
 
 	dev->dev_ops = &sfc_eth_dev_ops;
 
 	return 0;
 
+fail_dp_tx:
+fail_kvarg_tx_datapath:
 fail_dp_rx:
 fail_kvarg_rx_datapath:
 	return rc;
@@ -1298,6 +1332,8 @@
 		/* Prefer EF10 datapath */
 		sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp);
 		sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
+
+		sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp);
 	}
 }
 
@@ -1431,6 +1467,7 @@
 RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio");
 RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
 	SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " "
+	SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " "
 	SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " "
 	SFC_KVARG_MCDI_LOGGING "=" SFC_KVARG_VALUES_BOOL " "
 	SFC_KVARG_DEBUG_INIT "=" SFC_KVARG_VALUES_BOOL);
diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c
index 64694f5..04e923f 100644
--- a/drivers/net/sfc/sfc_ev.c
+++ b/drivers/net/sfc/sfc_ev.c
@@ -184,16 +184,18 @@
 sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id)
 {
 	struct sfc_evq *evq = arg;
-	struct sfc_txq *txq;
+	struct sfc_dp_txq *dp_txq;
+	struct sfc_efx_txq *txq;
 	unsigned int stop;
 	unsigned int delta;
 
-	txq = evq->txq;
+	dp_txq = evq->dp_txq;
+	SFC_ASSERT(dp_txq != NULL);
 
-	SFC_ASSERT(txq != NULL);
+	txq = sfc_efx_txq_by_dp_txq(dp_txq);
 	SFC_ASSERT(txq->evq == evq);
 
-	if (unlikely((txq->state & SFC_TXQ_STARTED) == 0))
+	if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_STARTED) == 0))
 		goto done;
 
 	stop = (id + 1) & txq->ptr_mask;
@@ -306,9 +308,13 @@
 sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index)
 {
 	struct sfc_evq *evq = arg;
+	struct sfc_dp_txq *dp_txq;
 	struct sfc_txq *txq;
 
-	txq = evq->txq;
+	dp_txq = evq->dp_txq;
+	SFC_ASSERT(dp_txq != NULL);
+
+	txq = dp_txq->ops->get_ctrl(dp_txq);
 	SFC_ASSERT(txq != NULL);
 	SFC_ASSERT(txq->hw_index == txq_hw_index);
 	SFC_ASSERT(txq->evq == evq);
@@ -442,7 +448,7 @@
 	.eec_link_change	= sfc_ev_nop_link_change,
 };
 
-static const efx_ev_callbacks_t sfc_ev_callbacks_tx = {
+static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = {
 	.eec_initialized	= sfc_ev_initialized,
 	.eec_rx			= sfc_ev_nop_rx,
 	.eec_tx			= sfc_ev_tx,
@@ -457,6 +463,21 @@
 	.eec_link_change	= sfc_ev_nop_link_change,
 };
 
+static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = {
+	.eec_initialized	= sfc_ev_initialized,
+	.eec_rx			= sfc_ev_nop_rx,
+	.eec_tx			= sfc_ev_nop_tx,
+	.eec_exception		= sfc_ev_exception,
+	.eec_rxq_flush_done	= sfc_ev_nop_rxq_flush_done,
+	.eec_rxq_flush_failed	= sfc_ev_nop_rxq_flush_failed,
+	.eec_txq_flush_done	= sfc_ev_txq_flush_done,
+	.eec_software		= sfc_ev_software,
+	.eec_sram		= sfc_ev_sram,
+	.eec_wake_up		= sfc_ev_wake_up,
+	.eec_timer		= sfc_ev_timer,
+	.eec_link_change	= sfc_ev_nop_link_change,
+};
+
 
 void
 sfc_ev_qpoll(struct sfc_evq *evq)
@@ -490,8 +511,12 @@
 					rxq_sw_index);
 		}
 
-		if (evq->txq != NULL) {
-			unsigned int txq_sw_index = sfc_txq_sw_index(evq->txq);
+		if (evq->dp_txq != NULL) {
+			struct sfc_txq *txq;
+			unsigned int txq_sw_index;
+
+			txq = evq->dp_txq->ops->get_ctrl(evq->dp_txq);
+			txq_sw_index = sfc_txq_sw_index(txq);
 
 			sfc_warn(sa,
 				 "restart TxQ %u because of exception on its EvQ %u",
@@ -561,14 +586,17 @@
 	if (rc != 0)
 		goto fail_ev_qcreate;
 
-	SFC_ASSERT(evq->dp_rxq == NULL || evq->txq == NULL);
+	SFC_ASSERT(evq->dp_rxq == NULL || evq->dp_txq == NULL);
 	if (evq->dp_rxq != 0) {
 		if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0)
 			evq->callbacks = &sfc_ev_callbacks_efx_rx;
 		else
 			evq->callbacks = &sfc_ev_callbacks_dp_rx;
-	} else if (evq->txq != 0) {
-		evq->callbacks = &sfc_ev_callbacks_tx;
+	} else if (evq->dp_txq != 0) {
+		if (strcmp(sa->dp_tx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0)
+			evq->callbacks = &sfc_ev_callbacks_efx_tx;
+		else
+			evq->callbacks = &sfc_ev_callbacks_dp_tx;
 	} else {
 		evq->callbacks = &sfc_ev_callbacks;
 	}
diff --git a/drivers/net/sfc/sfc_ev.h b/drivers/net/sfc/sfc_ev.h
index e99cd74..f7434a3 100644
--- a/drivers/net/sfc/sfc_ev.h
+++ b/drivers/net/sfc/sfc_ev.h
@@ -30,8 +30,12 @@
 #ifndef _SFC_EV_H_
 #define _SFC_EV_H_
 
+#include <rte_ethdev.h>
+
 #include "efx.h"
 
+#include "sfc.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -41,7 +45,7 @@
 
 struct sfc_adapter;
 struct sfc_dp_rxq;
-struct sfc_txq;
+struct sfc_dp_txq;
 
 enum sfc_evq_state {
 	SFC_EVQ_UNINITIALIZED = 0,
@@ -60,7 +64,7 @@ struct sfc_evq {
 	boolean_t			exception;
 	efsys_mem_t			mem;
 	struct sfc_dp_rxq		*dp_rxq;
-	struct sfc_txq			*txq;
+	struct sfc_dp_txq		*dp_txq;
 
 	/* Not used on datapath */
 	struct sfc_adapter		*sa;
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index d8529fa..f23bc1a 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -46,6 +46,7 @@
 		SFC_KVARG_MCDI_LOGGING,
 		SFC_KVARG_PERF_PROFILE,
 		SFC_KVARG_RX_DATAPATH,
+		SFC_KVARG_TX_DATAPATH,
 		NULL,
 	};
 
diff --git a/drivers/net/sfc/sfc_kvargs.h b/drivers/net/sfc/sfc_kvargs.h
index b57439b..14f46db 100644
--- a/drivers/net/sfc/sfc_kvargs.h
+++ b/drivers/net/sfc/sfc_kvargs.h
@@ -60,6 +60,10 @@
 	"[" SFC_KVARG_DATAPATH_EFX "|" \
 	    SFC_KVARG_DATAPATH_EF10 "]"
 
+#define SFC_KVARG_TX_DATAPATH		"tx_datapath"
+#define SFC_KVARG_VALUES_TX_DATAPATH \
+	"[" SFC_KVARG_DATAPATH_EFX "]"
+
 struct sfc_adapter;
 
 int sfc_kvargs_parse(struct sfc_adapter *sa);
diff --git a/drivers/net/sfc/sfc_tso.c b/drivers/net/sfc/sfc_tso.c
index 68d84c9..317c805 100644
--- a/drivers/net/sfc/sfc_tso.c
+++ b/drivers/net/sfc/sfc_tso.c
@@ -42,13 +42,13 @@
 #define SFC_TSO_OPDESCS_IDX_SHIFT	2
 
 int
-sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
-			unsigned int txq_entries, unsigned int socket_id)
+sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+			    unsigned int txq_entries, unsigned int socket_id)
 {
 	unsigned int i;
 
 	for (i = 0; i < txq_entries; ++i) {
-		sw_ring[i].tsoh = rte_malloc_socket("sfc-txq-tsoh-obj",
+		sw_ring[i].tsoh = rte_malloc_socket("sfc-efx-txq-tsoh-obj",
 						    SFC_TSOH_STD_LEN,
 						    SFC_TX_SEG_BOUNDARY,
 						    socket_id);
@@ -66,7 +66,8 @@
 }
 
 void
-sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, unsigned int txq_entries)
+sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+			   unsigned int txq_entries)
 {
 	unsigned int i;
 
@@ -77,8 +78,8 @@
 }
 
 static void
-sfc_tso_prepare_header(struct sfc_txq *txq, struct rte_mbuf **in_seg,
-		       size_t *in_off, unsigned int idx, size_t bytes_left)
+sfc_efx_tso_prepare_header(struct sfc_efx_txq *txq, struct rte_mbuf **in_seg,
+			   size_t *in_off, unsigned int idx, size_t bytes_left)
 {
 	struct rte_mbuf *m = *in_seg;
 	size_t bytes_to_copy = 0;
@@ -109,9 +110,9 @@
 }
 
 int
-sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg,
-	   size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs,
-	   size_t *pkt_len)
+sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
+	       struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend,
+	       unsigned int *pkt_descs, size_t *pkt_len)
 {
 	uint8_t *tsoh;
 	const struct tcp_hdr *th;
@@ -151,7 +152,8 @@
 	 */
 	if ((m->data_len < header_len) ||
 	    ((paddr_next_frag - header_paddr) < header_len)) {
-		sfc_tso_prepare_header(txq, in_seg, in_off, idx, header_len);
+		sfc_efx_tso_prepare_header(txq, in_seg, in_off, idx,
+					   header_len);
 		tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
 
 		header_paddr = rte_malloc_virt2phy((void *)tsoh);
diff --git a/drivers/net/sfc/sfc_tx.c b/drivers/net/sfc/sfc_tx.c
index 5a6282c..31bccac 100644
--- a/drivers/net/sfc/sfc_tx.c
+++ b/drivers/net/sfc/sfc_tx.c
@@ -33,6 +33,7 @@
 #include "sfc_ev.h"
 #include "sfc_tx.h"
 #include "sfc_tweak.h"
+#include "sfc_kvargs.h"
 
 /*
  * Maximum number of TX queue flush attempts in case of
@@ -109,27 +110,29 @@
 	txq->state &= ~SFC_TXQ_FLUSHING;
 }
 
+static sfc_dp_exception_t sfc_tx_dp_exception;
 static void
-sfc_tx_reap(struct sfc_txq *txq)
+sfc_tx_dp_exception(void *ctrl)
 {
-	unsigned int    completed;
+	struct sfc_txq *txq = ctrl;
+	struct sfc_adapter *sa = txq->evq->sa;
+	unsigned int txq_sw_index;
+	int rc;
 
+	if (!sfc_adapter_trylock(sa))
+		return;
 
-	sfc_ev_qpoll(txq->evq);
-
-	for (completed = txq->completed;
-	     completed != txq->pending; completed++) {
-		struct sfc_tx_sw_desc *txd;
+	txq_sw_index = sfc_txq_sw_index(txq);
 
-		txd = &txq->sw_ring[completed & txq->ptr_mask];
+	sfc_warn(sa, "restart TxQ %u because of datapath exception",
+		 txq_sw_index);
 
-		if (txd->mbuf != NULL) {
-			rte_pktmbuf_free(txd->mbuf);
-			txd->mbuf = NULL;
-		}
-	}
+	sfc_tx_qstop(sa, txq_sw_index);
+	rc = sfc_tx_qstart(sa, txq_sw_index);
+	if (rc != 0)
+		sfc_err(sa, "cannot restart TxQ %u", txq_sw_index);
 
-	txq->completed = completed;
+	sfc_adapter_unlock(sa);
 }
 
 int
@@ -142,6 +145,7 @@
 	struct sfc_txq *txq;
 	unsigned int evq_index = sfc_evq_index_by_txq_sw_index(sa, sw_index);
 	int rc = 0;
+	struct sfc_dp_tx_qcreate_args args;
 
 	sfc_log_init(sa, "TxQ = %u", sw_index);
 
@@ -166,56 +170,43 @@
 	if (txq == NULL)
 		goto fail_txq_alloc;
 
+	txq_info->txq = txq;
+
+	txq->hw_index = sw_index;
+	txq->evq = evq;
+	txq->free_thresh =
+		(tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
+		SFC_TX_DEFAULT_FREE_THRESH;
+	txq->flags = tx_conf->txq_flags;
+
 	rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries),
 			   socket_id, &txq->mem);
 	if (rc != 0)
 		goto fail_dma_alloc;
 
-	rc = ENOMEM;
-	txq->pend_desc = rte_calloc_socket("sfc-txq-pend-desc",
-					   EFX_TXQ_LIMIT(txq_info->entries),
-					   sizeof(efx_desc_t), 0, socket_id);
-	if (txq->pend_desc == NULL)
-		goto fail_pend_desc_alloc;
+	memset(&args, 0, sizeof(args));
+	args.free_thresh = txq->free_thresh;
+	args.flags = tx_conf->txq_flags;
+	args.txq_entries = txq_info->entries;
 
-	rc = ENOMEM;
-	txq->sw_ring = rte_calloc_socket("sfc-txq-desc", txq_info->entries,
-					 sizeof(*txq->sw_ring), 0, socket_id);
-	if (txq->sw_ring == NULL)
-		goto fail_desc_alloc;
+	rc = sa->dp_tx->qcreate(txq, sfc_tx_dp_exception, socket_id, &args,
+				&txq->dp);
+	if (rc != 0)
+		goto fail_dp_tx_qinit;
 
-	if (sa->tso) {
-		rc = sfc_tso_alloc_tsoh_objs(txq->sw_ring, txq_info->entries,
-					     socket_id);
-		if (rc != 0)
-			goto fail_alloc_tsoh_objs;
-	}
+	evq->dp_txq = txq->dp;
 
 	txq->state = SFC_TXQ_INITIALIZED;
-	txq->ptr_mask = txq_info->entries - 1;
-	txq->free_thresh = (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
-						     SFC_TX_DEFAULT_FREE_THRESH;
-	txq->hw_index = sw_index;
-	txq->flags = tx_conf->txq_flags;
-	txq->evq = evq;
-
-	evq->txq = txq;
 
-	txq_info->txq = txq;
 	txq_info->deferred_start = (tx_conf->tx_deferred_start != 0);
 
 	return 0;
 
-fail_alloc_tsoh_objs:
-	rte_free(txq->sw_ring);
-
-fail_desc_alloc:
-	rte_free(txq->pend_desc);
-
-fail_pend_desc_alloc:
+fail_dp_tx_qinit:
 	sfc_dma_free(sa, &txq->mem);
 
 fail_dma_alloc:
+	txq_info->txq = NULL;
 	rte_free(txq);
 
 fail_txq_alloc:
@@ -244,13 +235,12 @@
 	SFC_ASSERT(txq != NULL);
 	SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED);
 
-	sfc_tso_free_tsoh_objs(txq->sw_ring, txq_info->entries);
+	sa->dp_tx->qdestroy(txq->dp);
+	txq->dp = NULL;
 
 	txq_info->txq = NULL;
 	txq_info->entries = 0;
 
-	rte_free(txq->sw_ring);
-	rte_free(txq->pend_desc);
 	sfc_dma_free(sa, &txq->mem);
 	rte_free(txq);
 }
@@ -405,12 +395,13 @@
 		goto fail_tx_qcreate;
 	}
 
-	txq->added = txq->pending = txq->completed = desc_index;
-	txq->hw_vlan_tci = 0;
-
 	efx_tx_qenable(txq->common);
 
-	txq->state |= (SFC_TXQ_STARTED | SFC_TXQ_RUNNING);
+	txq->state |= SFC_TXQ_STARTED;
+
+	rc = sa->dp_tx->qstart(txq->dp, evq->read_ptr, desc_index);
+	if (rc != 0)
+		goto fail_dp_qstart;
 
 	/*
 	 * It seems to be used by DPDK for debug purposes only ('rte_ether')
@@ -420,6 +411,10 @@
 
 	return 0;
 
+fail_dp_qstart:
+	txq->state = SFC_TXQ_INITIALIZED;
+	efx_tx_qdestroy(txq->common);
+
 fail_tx_qcreate:
 	sfc_ev_qstop(sa, evq->evq_index);
 
@@ -435,7 +430,6 @@
 	struct sfc_txq *txq;
 	unsigned int retry_count;
 	unsigned int wait_count;
-	unsigned int txds;
 
 	sfc_log_init(sa, "TxQ = %u", sw_index);
 
@@ -449,7 +443,7 @@
 
 	SFC_ASSERT(txq->state & SFC_TXQ_STARTED);
 
-	txq->state &= ~SFC_TXQ_RUNNING;
+	sa->dp_tx->qstop(txq->dp, &txq->evq->read_ptr);
 
 	/*
 	 * Retry TX queue flushing in case of flush failed or
@@ -484,14 +478,7 @@
 			sfc_info(sa, "TxQ %u flushed", sw_index);
 	}
 
-	sfc_tx_reap(txq);
-
-	for (txds = 0; txds < txq_info->entries; txds++) {
-		if (txq->sw_ring[txds].mbuf != NULL) {
-			rte_pktmbuf_free(txq->sw_ring[txds].mbuf);
-			txq->sw_ring[txds].mbuf = NULL;
-		}
-	}
+	sa->dp_tx->qreap(txq->dp);
 
 	txq->state = SFC_TXQ_INITIALIZED;
 
@@ -563,6 +550,28 @@
 	efx_tx_fini(sa->nic);
 }
 
+static void
+sfc_efx_tx_reap(struct sfc_efx_txq *txq)
+{
+	unsigned int completed;
+
+	sfc_ev_qpoll(txq->evq);
+
+	for (completed = txq->completed;
+	     completed != txq->pending; completed++) {
+		struct sfc_efx_tx_sw_desc *txd;
+
+		txd = &txq->sw_ring[completed & txq->ptr_mask];
+
+		if (txd->mbuf != NULL) {
+			rte_pktmbuf_free(txd->mbuf);
+			txd->mbuf = NULL;
+		}
+	}
+
+	txq->completed = completed;
+}
+
 /*
  * The function is used to insert or update VLAN tag;
  * the firmware has state of the firmware tag to insert per TxQ
@@ -571,8 +580,8 @@
  * the function will update it
  */
 static unsigned int
-sfc_tx_maybe_insert_tag(struct sfc_txq *txq, struct rte_mbuf *m,
-			efx_desc_t **pend)
+sfc_efx_tx_maybe_insert_tag(struct sfc_efx_txq *txq, struct rte_mbuf *m,
+			    efx_desc_t **pend)
 {
 	uint16_t this_tag = ((m->ol_flags & PKT_TX_VLAN_PKT) ?
 			     m->vlan_tci : 0);
@@ -594,10 +603,11 @@
 	return 1;
 }
 
-uint16_t
-sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+static uint16_t
+sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 {
-	struct sfc_txq *txq = (struct sfc_txq *)tx_queue;
+	struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue;
+	struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
 	unsigned int added = txq->added;
 	unsigned int pushed = added;
 	unsigned int pkts_sent = 0;
@@ -609,7 +619,7 @@
 	int rc __rte_unused;
 	struct rte_mbuf **pktp;
 
-	if (unlikely((txq->state & SFC_TXQ_RUNNING) == 0))
+	if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0))
 		goto done;
 
 	/*
@@ -620,7 +630,7 @@
 	reap_done = (fill_level > soft_max_fill);
 
 	if (reap_done) {
-		sfc_tx_reap(txq);
+		sfc_efx_tx_reap(txq);
 		/*
 		 * Recalculate fill level since 'txq->completed'
 		 * might have changed on reap
@@ -643,15 +653,16 @@
 		 * DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then
 		 * TX_ERROR will occur
 		 */
-		pkt_descs += sfc_tx_maybe_insert_tag(txq, m_seg, &pend);
+		pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend);
 
+#ifdef RTE_LIBRTE_SFC_EFX_TSO
 		if (m_seg->ol_flags & PKT_TX_TCP_SEG) {
 			/*
 			 * We expect correct 'pkt->l[2, 3, 4]_len' values
 			 * to be set correctly by the caller
 			 */
-			if (sfc_tso_do(txq, added, &m_seg, &in_off, &pend,
-				       &pkt_descs, &pkt_len) != 0) {
+			if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend,
+					   &pkt_descs, &pkt_len) != 0) {
 				/* We may have reached this place for
 				 * one of the following reasons:
 				 *
@@ -682,6 +693,7 @@
 			 * as for the usual non-TSO path
 			 */
 		}
+#endif /* RTE_LIBRTE_SFC_EFX_TSO */
 
 		for (; m_seg != NULL; m_seg = m_seg->next) {
 			efsys_dma_addr_t	next_frag;
@@ -729,7 +741,7 @@
 			 * Try to reap (if we haven't yet).
 			 */
 			if (!reap_done) {
-				sfc_tx_reap(txq);
+				sfc_efx_tx_reap(txq);
 				reap_done = B_TRUE;
 				fill_level = added - txq->completed;
 				if (fill_level > hard_max_fill) {
@@ -758,9 +770,143 @@
 
 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
 	if (!reap_done)
-		sfc_tx_reap(txq);
+		sfc_efx_tx_reap(txq);
 #endif
 
 done:
 	return pkts_sent;
 }
+
+static sfc_dp_tx_qcreate_t sfc_efx_tx_qcreate;
+static int
+sfc_efx_tx_qcreate(void *ctrl, __rte_unused sfc_dp_exception_t *exception,
+		   int socket_id, const struct sfc_dp_tx_qcreate_args *args,
+		   struct sfc_dp_txq **dp_txqp)
+{
+	struct sfc_txq *ctrl_txq = ctrl;
+	struct sfc_efx_txq *txq;
+	int rc;
+
+	rc = ENOMEM;
+	txq = rte_zmalloc_socket("sfc-efx-txq", sizeof(*txq),
+				 RTE_CACHE_LINE_SIZE, socket_id);
+	if (txq == NULL)
+		goto fail_txq_alloc;
+
+	rc = ENOMEM;
+	txq->pend_desc = rte_calloc_socket("sfc-efx-txq-pend-desc",
+					   EFX_TXQ_LIMIT(args->txq_entries),
+					   sizeof(*txq->pend_desc), 0,
+					   socket_id);
+	if (txq->pend_desc == NULL)
+		goto fail_pend_desc_alloc;
+
+	rc = ENOMEM;
+	txq->sw_ring = rte_calloc_socket("sfc-efx-txq-sw_ring",
+					 args->txq_entries,
+					 sizeof(*txq->sw_ring),
+					 RTE_CACHE_LINE_SIZE, socket_id);
+	if (txq->sw_ring == NULL)
+		goto fail_sw_ring_alloc;
+
+	if (ctrl_txq->evq->sa->tso) {
+		rc = sfc_efx_tso_alloc_tsoh_objs(txq->sw_ring,
+						 args->txq_entries, socket_id);
+		if (rc != 0)
+			goto fail_alloc_tsoh_objs;
+	}
+
+	txq->ctrl = ctrl_txq;
+	txq->evq = ctrl_txq->evq;
+	txq->ptr_mask = args->txq_entries - 1;
+	txq->free_thresh = args->free_thresh;
+
+	*dp_txqp = &txq->dp;
+	return 0;
+
+fail_alloc_tsoh_objs:
+	rte_free(txq->sw_ring);
+
+fail_sw_ring_alloc:
+	rte_free(txq->pend_desc);
+
+fail_pend_desc_alloc:
+	rte_free(txq);
+
+fail_txq_alloc:
+	return rc;
+}
+
+static sfc_dp_tx_qdestroy_t sfc_efx_tx_qdestroy;
+static void
+sfc_efx_tx_qdestroy(struct sfc_dp_txq *dp_txq)
+{
+	struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+
+	sfc_efx_tso_free_tsoh_objs(txq->sw_ring, txq->ptr_mask + 1);
+	rte_free(txq->sw_ring);
+	rte_free(txq->pend_desc);
+	rte_free(txq);
+}
+
+static sfc_dp_tx_qstart_t sfc_efx_tx_qstart;
+static int
+sfc_efx_tx_qstart(struct sfc_dp_txq *dp_txq,
+		  __rte_unused unsigned int evq_read_ptr,
+		  unsigned int txq_desc_index)
+{
+	/* libefx-based datapath is specific to libefx-based PMD */
+	struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+
+	txq->common = txq->ctrl->common;
+
+	txq->pending = txq->completed = txq->added = txq_desc_index;
+	txq->hw_vlan_tci = 0;
+
+	txq->flags |= (SFC_EFX_TXQ_FLAG_STARTED | SFC_EFX_TXQ_FLAG_RUNNING);
+
+	return 0;
+}
+
+static sfc_dp_tx_qstop_t sfc_efx_tx_qstop;
+static void
+sfc_efx_tx_qstop(struct sfc_dp_txq *dp_txq,
+		 __rte_unused unsigned int *evq_read_ptr)
+{
+	struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+
+	txq->flags &= ~SFC_EFX_TXQ_FLAG_RUNNING;
+}
+
+static sfc_dp_tx_qreap_t sfc_efx_tx_qreap;
+static void
+sfc_efx_tx_qreap(struct sfc_dp_txq *dp_txq)
+{
+	struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+	unsigned int txds;
+
+	sfc_efx_tx_reap(txq);
+
+	for (txds = 0; txds <= txq->ptr_mask; txds++) {
+		if (txq->sw_ring[txds].mbuf != NULL) {
+			rte_pktmbuf_free(txq->sw_ring[txds].mbuf);
+			txq->sw_ring[txds].mbuf = NULL;
+		}
+	}
+
+	txq->flags &= ~SFC_EFX_TXQ_FLAG_STARTED;
+}
+
+struct sfc_dp_tx sfc_efx_tx = {
+	.dp = {
+		.name		= SFC_KVARG_DATAPATH_EFX,
+		.type		= SFC_DP_TX,
+		.hw_fw_caps	= 0,
+	},
+	.qcreate		= sfc_efx_tx_qcreate,
+	.qdestroy		= sfc_efx_tx_qdestroy,
+	.qstart			= sfc_efx_tx_qstart,
+	.qstop			= sfc_efx_tx_qstop,
+	.qreap			= sfc_efx_tx_qreap,
+	.pkt_burst		= sfc_efx_xmit_pkts,
+};
diff --git a/drivers/net/sfc/sfc_tx.h b/drivers/net/sfc/sfc_tx.h
index 39977a5..63c86a1 100644
--- a/drivers/net/sfc/sfc_tx.h
+++ b/drivers/net/sfc/sfc_tx.h
@@ -35,6 +35,8 @@
 
 #include "efx.h"
 
+#include "sfc_dp_tx.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -48,7 +50,11 @@
 struct sfc_adapter;
 struct sfc_evq;
 
-struct sfc_tx_sw_desc {
+/**
+ * Software Tx descriptor information associated with hardware Tx
+ * descriptor.
+ */
+struct sfc_efx_tx_sw_desc {
 	struct rte_mbuf		*mbuf;
 	uint8_t			*tsoh;	/* Buffer to store TSO header */
 };
@@ -58,36 +64,69 @@ enum sfc_txq_state_bit {
 #define SFC_TXQ_INITIALIZED	(1 << SFC_TXQ_INITIALIZED_BIT)
 	SFC_TXQ_STARTED_BIT,
 #define SFC_TXQ_STARTED		(1 << SFC_TXQ_STARTED_BIT)
-	SFC_TXQ_RUNNING_BIT,
-#define SFC_TXQ_RUNNING		(1 << SFC_TXQ_RUNNING_BIT)
 	SFC_TXQ_FLUSHING_BIT,
 #define SFC_TXQ_FLUSHING	(1 << SFC_TXQ_FLUSHING_BIT)
 	SFC_TXQ_FLUSHED_BIT,
 #define SFC_TXQ_FLUSHED		(1 << SFC_TXQ_FLUSHED_BIT)
 };
 
+/**
+ * Transmit queue control information. Not used on datapath.
+ * Allocated on the socket specified on the queue setup.
+ */
 struct sfc_txq {
-	struct sfc_evq		*evq;
-	struct sfc_tx_sw_desc	*sw_ring;
-	unsigned int		state;
-	unsigned int		ptr_mask;
-	efx_desc_t		*pend_desc;
-	efx_txq_t		*common;
-	efsys_mem_t		mem;
-	unsigned int		added;
-	unsigned int		pending;
-	unsigned int		completed;
-	unsigned int		free_thresh;
-	uint16_t		hw_vlan_tci;
-
-	unsigned int		hw_index;
-	unsigned int		flags;
+	unsigned int			state;
+	unsigned int			hw_index;
+	struct sfc_evq			*evq;
+	efsys_mem_t			mem;
+	struct sfc_dp_txq		*dp;
+	efx_txq_t			*common;
+	unsigned int			free_thresh;
+	unsigned int			flags;
 };
 
 static inline unsigned int
+sfc_txq_sw_index_by_hw_index(unsigned int hw_index)
+{
+	return hw_index;
+}
+
+static inline unsigned int
 sfc_txq_sw_index(const struct sfc_txq *txq)
 {
-	return txq->hw_index;
+	return sfc_txq_sw_index_by_hw_index(txq->hw_index);
+}
+
+/**
+ * Transmit queue information used on libefx-based data path.
+ * Allocated on the socket specified on the queue setup.
+ */
+struct sfc_efx_txq {
+	struct sfc_evq			*evq;
+	struct sfc_efx_tx_sw_desc	*sw_ring;
+	unsigned int			ptr_mask;
+	efx_desc_t			*pend_desc;
+	efx_txq_t			*common;
+	unsigned int			added;
+	unsigned int			pending;
+	unsigned int			completed;
+	unsigned int			free_thresh;
+	uint16_t			hw_vlan_tci;
+
+	unsigned int			hw_index;
+	unsigned int			flags;
+#define SFC_EFX_TXQ_FLAG_STARTED	0x1
+#define SFC_EFX_TXQ_FLAG_RUNNING	0x2
+
+	/* Datapath transmit queue anchor */
+	struct sfc_dp_txq		dp;
+	struct sfc_txq			*ctrl;
+};
+
+static inline struct sfc_efx_txq *
+sfc_efx_txq_by_dp_txq(struct sfc_dp_txq *dp_txq)
+{
+	return container_of(dp_txq, struct sfc_efx_txq, dp);
 }
 
 struct sfc_txq_info {
@@ -111,17 +150,15 @@ int sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
 int sfc_tx_start(struct sfc_adapter *sa);
 void sfc_tx_stop(struct sfc_adapter *sa);
 
-uint16_t sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
-		       uint16_t nb_pkts);
-
 /* From 'sfc_tso.c' */
-int sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
-			    unsigned int txq_entries, unsigned int socket_id);
-void sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
-			    unsigned int txq_entries);
-int sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg,
-	       size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs,
-	       size_t *pkt_len);
+int sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+				unsigned int txq_entries,
+				unsigned int socket_id);
+void sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+				unsigned int txq_entries);
+int sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
+		   struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend,
+		   unsigned int *pkt_descs, size_t *pkt_len);
 
 #ifdef __cplusplus
 }
-- 
1.8.2.3



More information about the dev mailing list