[dpdk-dev] [PATCH 04/13] net/sfc: factor out libefx-based Rx datapath

Andrew Rybchenko arybchenko at solarflare.com
Thu Mar 2 08:07:10 CET 2017


Split control and datapath to make datapath substitutable and
possibly reusable with alternative control path.

libefx-based Rx datapath is bound to libefx control path, but
other datapaths should be possible to use with alternative
control path(s).

Signed-off-by: Andrew Rybchenko <arybchenko at solarflare.com>
---
 doc/guides/nics/sfc_efx.rst  |   7 +
 drivers/net/sfc/Makefile     |   1 +
 drivers/net/sfc/sfc.h        |   3 +
 drivers/net/sfc/sfc_dp.c     |  87 ++++++++++++
 drivers/net/sfc/sfc_dp.h     |  86 ++++++++++++
 drivers/net/sfc/sfc_dp_rx.h  | 184 +++++++++++++++++++++++++
 drivers/net/sfc/sfc_ethdev.c | 105 ++++++++++++---
 drivers/net/sfc/sfc_ev.c     |  75 ++++++++---
 drivers/net/sfc/sfc_ev.h     |   4 +-
 drivers/net/sfc/sfc_kvargs.c |  10 ++
 drivers/net/sfc/sfc_kvargs.h |   8 ++
 drivers/net/sfc/sfc_rx.c     | 313 ++++++++++++++++++++++++++++++++++---------
 drivers/net/sfc/sfc_rx.h     |  73 ++++++----
 13 files changed, 824 insertions(+), 132 deletions(-)
 create mode 100644 drivers/net/sfc/sfc_dp.c
 create mode 100644 drivers/net/sfc/sfc_dp.h
 create mode 100644 drivers/net/sfc/sfc_dp_rx.h

diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst
index 0a05a0a..8bd8a0c 100644
--- a/doc/guides/nics/sfc_efx.rst
+++ b/doc/guides/nics/sfc_efx.rst
@@ -181,6 +181,13 @@ whitelist option like "-w 02:00.0,arg1=value1,...".
 Case-insensitive 1/y/yes/on or 0/n/no/off may be used to specify
 boolean parameters value.
 
+- ``rx_datapath`` [auto|efx] (default **auto**)
+
+  Choose receive datapath implementation.
+  **auto** allows the driver itself to make a choice based on firmware
+  features available and required by the datapath implementation.
+  **efx** chooses libefx-based datapath which supports Rx scatter.
+
 - ``perf_profile`` [auto|throughput|low-latency] (default **throughput**)
 
   Choose hardware tunning to be optimized for either throughput or
diff --git a/drivers/net/sfc/Makefile b/drivers/net/sfc/Makefile
index 619a0ed..befff4c 100644
--- a/drivers/net/sfc/Makefile
+++ b/drivers/net/sfc/Makefile
@@ -90,6 +90,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_port.c
 SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_rx.c
 SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tx.c
 SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tso.c
+SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_dp.c
 
 VPATH += $(SRCDIR)/base
 
diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
index 8c6c02f..2512f2e 100644
--- a/drivers/net/sfc/sfc.h
+++ b/drivers/net/sfc/sfc.h
@@ -136,6 +136,7 @@ struct sfc_intr {
 struct sfc_evq_info;
 struct sfc_rxq_info;
 struct sfc_txq_info;
+struct sfc_dp_rx;
 
 struct sfc_port {
 	unsigned int			lsc_seq;
@@ -209,6 +210,8 @@ struct sfc_adapter {
 	unsigned int			rss_tbl[EFX_RSS_TBL_SIZE];
 	uint8_t				rss_key[SFC_RSS_KEY_SIZE];
 #endif
+
+	const struct sfc_dp_rx		*dp_rx;
 };
 
 /*
diff --git a/drivers/net/sfc/sfc_dp.c b/drivers/net/sfc/sfc_dp.c
new file mode 100644
index 0000000..c4d5fb3
--- /dev/null
+++ b/drivers/net/sfc/sfc_dp.c
@@ -0,0 +1,87 @@
+/*-
+ * Copyright (c) 2017 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/queue.h>
+#include <string.h>
+#include <errno.h>
+
+#include <rte_log.h>
+
+#include "sfc_dp.h"
+
+struct sfc_dp *
+sfc_dp_find_by_name(struct sfc_dp_list *head, enum sfc_dp_type type,
+		    const char *name)
+{
+	struct sfc_dp *entry;
+
+	TAILQ_FOREACH(entry, head, links) {
+		if (entry->type != type)
+			continue;
+
+		if (strcmp(entry->name, name) == 0)
+			return entry;
+	}
+
+	return NULL;
+}
+
+struct sfc_dp *
+sfc_dp_find_by_caps(struct sfc_dp_list *head, enum sfc_dp_type type,
+		    unsigned int avail_caps)
+{
+	struct sfc_dp *entry;
+
+	TAILQ_FOREACH(entry, head, links) {
+		if (entry->type != type)
+			continue;
+
+		/* Take the first matching */
+		if (sfc_dp_match_hw_fw_caps(entry, avail_caps))
+			return entry;
+	}
+
+	return NULL;
+}
+
+int
+sfc_dp_register(struct sfc_dp_list *head, struct sfc_dp *entry)
+{
+	if (sfc_dp_find_by_name(head, entry->type, entry->name) != NULL) {
+		rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD,
+			"sfc %s dapapath '%s' already registered\n",
+			entry->type == SFC_DP_RX ? "Rx" : "unknown",
+			entry->name);
+		return EEXIST;
+	}
+
+	TAILQ_INSERT_TAIL(head, entry, links);
+
+	return 0;
+}
diff --git a/drivers/net/sfc/sfc_dp.h b/drivers/net/sfc/sfc_dp.h
new file mode 100644
index 0000000..8f78f98
--- /dev/null
+++ b/drivers/net/sfc/sfc_dp.h
@@ -0,0 +1,86 @@
+/*-
+ * Copyright (c) 2017 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SFC_DP_H
+#define _SFC_DP_H
+
+#include <stdbool.h>
+#include <sys/queue.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SFC_DIV_ROUND_UP(a, b) \
+	__extension__ ({		\
+		typeof(a) _a = (a);	\
+		typeof(b) _b = (b);	\
+					\
+		(_a + (_b - 1)) / _b;	\
+	})
+
+/**
+ * Datapath exception handler to be provided by the control path.
+ */
+typedef void (sfc_dp_exception_t)(void *ctrl);
+
+enum sfc_dp_type {
+	SFC_DP_RX = 0,	/**< Receive datapath */
+};
+
+/** Datapath definition */
+struct sfc_dp {
+	TAILQ_ENTRY(sfc_dp)		links;
+	const char			*name;
+	enum sfc_dp_type		type;
+	/* Mask of required hardware/firmware capabilities */
+	unsigned int			hw_fw_caps;
+};
+
+/** List of datapath variants */
+TAILQ_HEAD(sfc_dp_list, sfc_dp);
+
+/* Check if available HW/FW capabilities are sufficient for the datapath */
+static inline bool
+sfc_dp_match_hw_fw_caps(const struct sfc_dp *dp, unsigned int avail_caps)
+{
+	return (dp->hw_fw_caps & avail_caps) == dp->hw_fw_caps;
+}
+
+struct sfc_dp *sfc_dp_find_by_name(struct sfc_dp_list *head,
+				   enum sfc_dp_type type, const char *name);
+struct sfc_dp *sfc_dp_find_by_caps(struct sfc_dp_list *head,
+				   enum sfc_dp_type type,
+				   unsigned int avail_caps);
+int sfc_dp_register(struct sfc_dp_list *head, struct sfc_dp *entry);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_DP_H */
diff --git a/drivers/net/sfc/sfc_dp_rx.h b/drivers/net/sfc/sfc_dp_rx.h
new file mode 100644
index 0000000..5a714a1
--- /dev/null
+++ b/drivers/net/sfc/sfc_dp_rx.h
@@ -0,0 +1,184 @@
+/*-
+ * Copyright (c) 2017 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SFC_DP_RX_H
+#define _SFC_DP_RX_H
+
+#include <rte_mempool.h>
+#include <rte_ethdev.h>
+
+#include "sfc_dp.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sfc_dp_rxq;
+
+/**
+ * Callback to get control path receive queue by datapath receive queue
+ * handle.
+ */
+typedef void * (sfc_dp_rxq_get_ctrl_t)(struct sfc_dp_rxq *dp_rxq);
+
+/** Datapath receive queue operations */
+struct sfc_dp_rxq_ops {
+	sfc_dp_rxq_get_ctrl_t		*get_ctrl;
+};
+
+/**
+ * Generic receive queue information used on data path.
+ * It must be kept as small as it is possible since it is built into
+ * the structure used on datapath.
+ */
+struct sfc_dp_rxq {
+	const struct sfc_dp_rxq_ops	*ops;
+};
+
+/**
+ * Datapath receive queue creation arguments.
+ *
+ * The structure is used just to pass information from control path to
+ * datapath. It could be just function arguments, but it would be hardly
+ * readable.
+ */
+struct sfc_dp_rx_qcreate_args {
+	/** Memory pool to allocate Rx buffer from */
+	struct rte_mempool	*refill_mb_pool;
+	/** Minimum number of unused Rx descriptors to do refill */
+	unsigned int		refill_threshold;
+	/**
+	 * Usable mbuf data space in accordance with alignment and
+	 * padding requirements imposed by HW.
+	 */
+	unsigned int		buf_size;
+	/** Port number to be set in the mbuf */
+	uint8_t			port_id;
+
+	/**
+	 * Maximum number of Rx descriptors completed in one Rx event.
+	 * Just for sanity checks if datapath would like to do.
+	 */
+	unsigned int		batch_max;
+
+	/** Pseudo-header size */
+	unsigned int		prefix_size;
+
+	/** Receive queue flags initializer */
+	unsigned int		flags;
+#define SFC_RXQ_FLAG_RSS_HASH	0x1
+
+	/** Rx queue size */
+	unsigned int		rxq_entries;
+};
+
+/**
+ * Allocate and initalize datapath receive queue.
+ *
+ * @param ctrl		Control path Rx queue opaque handle
+ * @param exception	Datapath exception handler to bail out to control path
+ * @param socket_id	Socket ID to allocate memory
+ * @param args		Function arguments wrapped in structure
+ * @param dp_rxqp	Location for generic datapath receive queue pointer
+ *
+ * @return 0 or positive errno.
+ */
+typedef int (sfc_dp_rx_qcreate_t)(void *ctrl, sfc_dp_exception_t *exception,
+				  int socket_id,
+				  const struct sfc_dp_rx_qcreate_args *args,
+				  struct sfc_dp_rxq **dp_rxqp);
+
+/**
+ * Free resources allocated for datapath recevie queue.
+ */
+typedef void (sfc_dp_rx_qdestroy_t)(struct sfc_dp_rxq *dp_rxq);
+
+/**
+ * Receive queue start callback.
+ *
+ * It handovers EvQ to the datapath.
+ */
+typedef int (sfc_dp_rx_qstart_t)(struct sfc_dp_rxq *dp_rxq,
+				 unsigned int evq_read_ptr);
+
+/**
+ * Receive queue stop function called before flush.
+ */
+typedef void (sfc_dp_rx_qstop_t)(struct sfc_dp_rxq *dp_rxq,
+				 unsigned int *evq_read_ptr);
+
+/**
+ * Receive queue purge function called after queue flush.
+ *
+ * Should be used to free unused recevie buffers.
+ */
+typedef void (sfc_dp_rx_qpurge_t)(struct sfc_dp_rxq *dp_rxq);
+
+/** Get packet types recognized/classified */
+typedef const uint32_t * (sfc_dp_rx_supported_ptypes_get_t)(void);
+
+/** Get number of pending Rx descriptors */
+typedef unsigned int (sfc_dp_rx_qdesc_npending_t)(struct sfc_dp_rxq *dp_rxq);
+
+/** Receive datapath definition */
+struct sfc_dp_rx {
+	struct sfc_dp				dp;
+
+	sfc_dp_rx_qcreate_t			*qcreate;
+	sfc_dp_rx_qdestroy_t			*qdestroy;
+	sfc_dp_rx_qstart_t			*qstart;
+	sfc_dp_rx_qstop_t			*qstop;
+	sfc_dp_rx_qpurge_t			*qpurge;
+	sfc_dp_rx_supported_ptypes_get_t	*supported_ptypes_get;
+	sfc_dp_rx_qdesc_npending_t		*qdesc_npending;
+	eth_rx_burst_t				pkt_burst;
+};
+
+static inline struct sfc_dp_rx *
+sfc_dp_find_rx_by_name(struct sfc_dp_list *head, const char *name)
+{
+	struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_RX, name);
+
+	return (p == NULL) ? NULL : container_of(p, struct sfc_dp_rx, dp);
+}
+
+static inline struct sfc_dp_rx *
+sfc_dp_find_rx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps)
+{
+	struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_RX, avail_caps);
+
+	return (p == NULL) ? NULL : container_of(p, struct sfc_dp_rx, dp);
+}
+
+extern struct sfc_dp_rx sfc_efx_rx;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_DP_RX_H */
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index 71587fb..3207cf4 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -30,6 +30,7 @@
 #include <rte_dev.h>
 #include <rte_ethdev.h>
 #include <rte_pci.h>
+#include <rte_errno.h>
 
 #include "efx.h"
 
@@ -40,7 +41,11 @@
 #include "sfc_ev.h"
 #include "sfc_rx.h"
 #include "sfc_tx.h"
+#include "sfc_dp.h"
+#include "sfc_dp_rx.h"
 
+static struct sfc_dp_list sfc_dp_head =
+	TAILQ_HEAD_INITIALIZER(sfc_dp_head);
 
 static void
 sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
@@ -114,19 +119,9 @@
 static const uint32_t *
 sfc_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 {
-	static const uint32_t ptypes[] = {
-		RTE_PTYPE_L2_ETHER,
-		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
-		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
-		RTE_PTYPE_L4_TCP,
-		RTE_PTYPE_L4_UDP,
-		RTE_PTYPE_UNKNOWN
-	};
-
-	if (dev->rx_pkt_burst == sfc_recv_pkts)
-		return ptypes;
-
-	return NULL;
+	struct sfc_adapter *sa = dev->data->dev_private;
+
+	return sa->dp_rx->supported_ptypes_get();
 }
 
 static int
@@ -366,7 +361,7 @@
 	if (rc != 0)
 		goto fail_rx_qinit;
 
-	dev->data->rx_queues[rx_queue_id] = sa->rxq_info[rx_queue_id].rxq;
+	dev->data->rx_queues[rx_queue_id] = sa->rxq_info[rx_queue_id].rxq->dp;
 
 	sfc_adapter_unlock(sa);
 
@@ -381,13 +376,15 @@
 static void
 sfc_rx_queue_release(void *queue)
 {
-	struct sfc_rxq *rxq = queue;
+	struct sfc_dp_rxq *dp_rxq = queue;
+	struct sfc_rxq *rxq;
 	struct sfc_adapter *sa;
 	unsigned int sw_index;
 
-	if (rxq == NULL)
+	if (dp_rxq == NULL)
 		return;
 
+	rxq = dp_rxq->ops->get_ctrl(dp_rxq);
 	sa = rxq->evq->sa;
 	sfc_adapter_lock(sa);
 
@@ -905,9 +902,9 @@
 static int
 sfc_rx_descriptor_done(void *queue, uint16_t offset)
 {
-	struct sfc_rxq *rxq = queue;
+	struct sfc_dp_rxq *dp_rxq = queue;
 
-	return sfc_rx_qdesc_done(rxq, offset);
+	return sfc_rx_qdesc_done(dp_rxq, offset);
 }
 
 static int
@@ -1230,6 +1227,69 @@
 };
 
 static int
+sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
+{
+	struct sfc_adapter *sa = dev->data->dev_private;
+	unsigned int avail_caps = 0;
+	const char *rx_name = NULL;
+	int rc;
+
+	if (sa == NULL || sa->state == SFC_ADAPTER_UNINITIALIZED)
+		return -E_RTE_SECONDARY;
+
+	rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH,
+				sfc_kvarg_string_handler, &rx_name);
+	if (rc != 0)
+		goto fail_kvarg_rx_datapath;
+
+	if (rx_name != NULL) {
+		sa->dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, rx_name);
+		if (sa->dp_rx == NULL) {
+			sfc_err(sa, "Rx datapath %s not found", rx_name);
+			rc = ENOENT;
+			goto fail_dp_rx;
+		}
+		if (!sfc_dp_match_hw_fw_caps(&sa->dp_rx->dp, avail_caps)) {
+			sfc_err(sa,
+				"Insufficient Hw/FW capabilities to use Rx datapath %s",
+				rx_name);
+			rc = EINVAL;
+			goto fail_dp_rx;
+		}
+	} else {
+		sa->dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps);
+		if (sa->dp_rx == NULL) {
+			sfc_err(sa, "Rx datapath by caps %#x not found",
+				avail_caps);
+			rc = ENOENT;
+			goto fail_dp_rx;
+		}
+	}
+
+	sfc_info(sa, "use %s Rx datapath", sa->dp_rx->dp.name);
+
+	dev->rx_pkt_burst = sa->dp_rx->pkt_burst;
+
+	dev->tx_pkt_burst = sfc_xmit_pkts;
+
+	dev->dev_ops = &sfc_eth_dev_ops;
+
+	return 0;
+
+fail_dp_rx:
+fail_kvarg_rx_datapath:
+	return rc;
+}
+
+static void
+sfc_register_dp(void)
+{
+	/* Register once */
+	if (TAILQ_EMPTY(&sfc_dp_head))
+		sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
+}
+
+static int
 sfc_eth_dev_init(struct rte_eth_dev *dev)
 {
 	struct sfc_adapter *sa = dev->data->dev_private;
@@ -1238,6 +1298,8 @@
 	const efx_nic_cfg_t *encp;
 	const struct ether_addr *from;
 
+	sfc_register_dp();
+
 	/* Required for logging */
 	sa->eth_dev = dev;
 
@@ -1278,12 +1340,10 @@
 	from = (const struct ether_addr *)(encp->enc_mac_addr);
 	ether_addr_copy(from, &dev->data->mac_addrs[0]);
 
-	dev->dev_ops = &sfc_eth_dev_ops;
-	dev->rx_pkt_burst = &sfc_recv_pkts;
-	dev->tx_pkt_burst = &sfc_xmit_pkts;
-
 	sfc_adapter_unlock(sa);
 
+	sfc_eth_dev_set_ops(dev);
+
 	sfc_log_init(sa, "done");
 	return 0;
 
@@ -1358,6 +1418,7 @@
 RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio");
 RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
+	SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " "
 	SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " "
 	SFC_KVARG_MCDI_LOGGING "=" SFC_KVARG_VALUES_BOOL " "
 	SFC_KVARG_DEBUG_INIT "=" SFC_KVARG_VALUES_BOOL);
diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c
index 7f79c7e..dca3a81 100644
--- a/drivers/net/sfc/sfc_ev.c
+++ b/drivers/net/sfc/sfc_ev.c
@@ -81,25 +81,25 @@
 }
 
 static boolean_t
-sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
-	  uint32_t size, uint16_t flags)
+sfc_ev_efx_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
+	      uint32_t size, uint16_t flags)
 {
 	struct sfc_evq *evq = arg;
-	struct sfc_rxq *rxq;
+	struct sfc_efx_rxq *rxq;
 	unsigned int stop;
 	unsigned int pending_id;
 	unsigned int delta;
 	unsigned int i;
-	struct sfc_rx_sw_desc *rxd;
+	struct sfc_efx_rx_sw_desc *rxd;
 
 	if (unlikely(evq->exception))
 		goto done;
 
-	rxq = evq->rxq;
+	rxq = sfc_efx_rxq_by_dp_rxq(evq->dp_rxq);
 
 	SFC_ASSERT(rxq != NULL);
 	SFC_ASSERT(rxq->evq == evq);
-	SFC_ASSERT(rxq->flags & SFC_RXQ_FLAG_STARTED);
+	SFC_ASSERT(rxq->flags & SFC_EFX_RXQ_FLAG_STARTED);
 
 	stop = (id + 1) & rxq->ptr_mask;
 	pending_id = rxq->pending & rxq->ptr_mask;
@@ -117,7 +117,9 @@
 			sfc_err(evq->sa,
 				"EVQ %u RxQ %u invalid RX abort "
 				"(id=%#x size=%u flags=%#x); needs restart",
-				evq->evq_index, sfc_rxq_sw_index(rxq),
+				evq->evq_index,
+				sfc_rxq_sw_index_by_hw_index(
+					rxq->ctrl->hw_index),
 				id, size, flags);
 			goto done;
 		}
@@ -132,8 +134,9 @@
 		sfc_err(evq->sa,
 			"EVQ %u RxQ %u completion out of order "
 			"(id=%#x delta=%u flags=%#x); needs restart",
-			evq->evq_index, sfc_rxq_sw_index(rxq), id, delta,
-			flags);
+			evq->evq_index,
+			sfc_rxq_sw_index_by_hw_index(rxq->ctrl->hw_index),
+			id, delta, flags);
 
 		goto done;
 	}
@@ -231,9 +234,13 @@
 sfc_ev_rxq_flush_done(void *arg, __rte_unused uint32_t rxq_hw_index)
 {
 	struct sfc_evq *evq = arg;
+	struct sfc_dp_rxq *dp_rxq;
 	struct sfc_rxq *rxq;
 
-	rxq = evq->rxq;
+	dp_rxq = evq->dp_rxq;
+	SFC_ASSERT(dp_rxq != NULL);
+
+	rxq = dp_rxq->ops->get_ctrl(dp_rxq);
 	SFC_ASSERT(rxq != NULL);
 	SFC_ASSERT(rxq->hw_index == rxq_hw_index);
 	SFC_ASSERT(rxq->evq == evq);
@@ -256,9 +263,13 @@
 sfc_ev_rxq_flush_failed(void *arg, __rte_unused uint32_t rxq_hw_index)
 {
 	struct sfc_evq *evq = arg;
+	struct sfc_dp_rxq *dp_rxq;
 	struct sfc_rxq *rxq;
 
-	rxq = evq->rxq;
+	dp_rxq = evq->dp_rxq;
+	SFC_ASSERT(dp_rxq != NULL);
+
+	rxq = dp_rxq->ops->get_ctrl(dp_rxq);
 	SFC_ASSERT(rxq != NULL);
 	SFC_ASSERT(rxq->hw_index == rxq_hw_index);
 	SFC_ASSERT(rxq->evq == evq);
@@ -387,9 +398,24 @@
 	.eec_link_change	= sfc_ev_link_change,
 };
 
-static const efx_ev_callbacks_t sfc_ev_callbacks_rx = {
+static const efx_ev_callbacks_t sfc_ev_callbacks_efx_rx = {
 	.eec_initialized	= sfc_ev_initialized,
-	.eec_rx			= sfc_ev_rx,
+	.eec_rx			= sfc_ev_efx_rx,
+	.eec_tx			= sfc_ev_nop_tx,
+	.eec_exception		= sfc_ev_exception,
+	.eec_rxq_flush_done	= sfc_ev_rxq_flush_done,
+	.eec_rxq_flush_failed	= sfc_ev_rxq_flush_failed,
+	.eec_txq_flush_done	= sfc_ev_nop_txq_flush_done,
+	.eec_software		= sfc_ev_software,
+	.eec_sram		= sfc_ev_sram,
+	.eec_wake_up		= sfc_ev_wake_up,
+	.eec_timer		= sfc_ev_timer,
+	.eec_link_change	= sfc_ev_nop_link_change,
+};
+
+static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx = {
+	.eec_initialized	= sfc_ev_initialized,
+	.eec_rx			= sfc_ev_nop_rx,
 	.eec_tx			= sfc_ev_nop_tx,
 	.eec_exception		= sfc_ev_exception,
 	.eec_rxq_flush_done	= sfc_ev_rxq_flush_done,
@@ -432,9 +458,12 @@
 		struct sfc_adapter *sa = evq->sa;
 		int rc;
 
-		if ((evq->rxq != NULL) &&
-		    (evq->rxq->flags & SFC_RXQ_FLAG_RUNNING)) {
-			unsigned int rxq_sw_index = sfc_rxq_sw_index(evq->rxq);
+		if (evq->dp_rxq != NULL) {
+			struct sfc_rxq *rxq;
+			unsigned int rxq_sw_index;
+
+			rxq = evq->dp_rxq->ops->get_ctrl(evq->dp_rxq);
+			rxq_sw_index = sfc_rxq_sw_index(rxq);
 
 			sfc_warn(sa,
 				 "restart RxQ %u because of exception on its EvQ %u",
@@ -518,13 +547,17 @@
 	if (rc != 0)
 		goto fail_ev_qcreate;
 
-	SFC_ASSERT(evq->rxq == NULL || evq->txq == NULL);
-	if (evq->rxq != 0)
-		evq->callbacks = &sfc_ev_callbacks_rx;
-	else if (evq->txq != 0)
+	SFC_ASSERT(evq->dp_rxq == NULL || evq->txq == NULL);
+	if (evq->dp_rxq != 0) {
+		if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0)
+			evq->callbacks = &sfc_ev_callbacks_efx_rx;
+		else
+			evq->callbacks = &sfc_ev_callbacks_dp_rx;
+	} else if (evq->txq != 0) {
 		evq->callbacks = &sfc_ev_callbacks_tx;
-	else
+	} else {
 		evq->callbacks = &sfc_ev_callbacks;
+	}
 
 	evq->init_state = SFC_EVQ_STARTING;
 
diff --git a/drivers/net/sfc/sfc_ev.h b/drivers/net/sfc/sfc_ev.h
index 41a37f4..e99cd74 100644
--- a/drivers/net/sfc/sfc_ev.h
+++ b/drivers/net/sfc/sfc_ev.h
@@ -40,7 +40,7 @@
 #define SFC_MGMT_EVQ_ENTRIES	(EFX_EVQ_MINNEVS)
 
 struct sfc_adapter;
-struct sfc_rxq;
+struct sfc_dp_rxq;
 struct sfc_txq;
 
 enum sfc_evq_state {
@@ -59,7 +59,7 @@ struct sfc_evq {
 	unsigned int			read_ptr;
 	boolean_t			exception;
 	efsys_mem_t			mem;
-	struct sfc_rxq			*rxq;
+	struct sfc_dp_rxq		*dp_rxq;
 	struct sfc_txq			*txq;
 
 	/* Not used on datapath */
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 227a8db..d8529fa 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -45,6 +45,7 @@
 		SFC_KVARG_DEBUG_INIT,
 		SFC_KVARG_MCDI_LOGGING,
 		SFC_KVARG_PERF_PROFILE,
+		SFC_KVARG_RX_DATAPATH,
 		NULL,
 	};
 
@@ -110,3 +111,12 @@
 
 	return 0;
 }
+
+int
+sfc_kvarg_string_handler(__rte_unused const char *key,
+			 const char *value_str, void *opaque)
+{
+	*(const char **)opaque = value_str;
+
+	return 0;
+}
diff --git a/drivers/net/sfc/sfc_kvargs.h b/drivers/net/sfc/sfc_kvargs.h
index 2fea9c7..2d0ffde 100644
--- a/drivers/net/sfc/sfc_kvargs.h
+++ b/drivers/net/sfc/sfc_kvargs.h
@@ -52,6 +52,12 @@
 	    SFC_KVARG_PERF_PROFILE_THROUGHPUT "|" \
 	    SFC_KVARG_PERF_PROFILE_LOW_LATENCY "]"
 
+#define SFC_KVARG_DATAPATH_EFX		"efx"
+
+#define SFC_KVARG_RX_DATAPATH		"rx_datapath"
+#define SFC_KVARG_VALUES_RX_DATAPATH \
+	"[" SFC_KVARG_DATAPATH_EFX "]"
+
 struct sfc_adapter;
 
 int sfc_kvargs_parse(struct sfc_adapter *sa);
@@ -62,6 +68,8 @@ int sfc_kvargs_process(struct sfc_adapter *sa, const char *key_match,
 
 int sfc_kvarg_bool_handler(const char *key, const char *value_str,
 			   void *opaque);
+int sfc_kvarg_string_handler(const char *key, const char *value_str,
+			     void *opaque);
 
 #ifdef __cplusplus
 }
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index 61654fc..e56837d 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -36,6 +36,7 @@
 #include "sfc_log.h"
 #include "sfc_ev.h"
 #include "sfc_rx.h"
+#include "sfc_kvargs.h"
 #include "sfc_tweak.h"
 
 /*
@@ -72,7 +73,7 @@
 }
 
 static void
-sfc_rx_qrefill(struct sfc_rxq *rxq)
+sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq)
 {
 	unsigned int free_space;
 	unsigned int bulks;
@@ -81,7 +82,7 @@
 	unsigned int added = rxq->added;
 	unsigned int id;
 	unsigned int i;
-	struct sfc_rx_sw_desc *rxd;
+	struct sfc_efx_rx_sw_desc *rxd;
 	struct rte_mbuf *m;
 	uint8_t port_id = rxq->port_id;
 
@@ -135,7 +136,7 @@
 }
 
 static uint64_t
-sfc_rx_desc_flags_to_offload_flags(const unsigned int desc_flags)
+sfc_efx_rx_desc_flags_to_offload_flags(const unsigned int desc_flags)
 {
 	uint64_t mbuf_flags = 0;
 
@@ -174,7 +175,7 @@
 }
 
 static uint32_t
-sfc_rx_desc_flags_to_packet_type(const unsigned int desc_flags)
+sfc_efx_rx_desc_flags_to_packet_type(const unsigned int desc_flags)
 {
 	return RTE_PTYPE_L2_ETHER |
 		((desc_flags & EFX_PKT_IPV4) ?
@@ -185,14 +186,30 @@
 		((desc_flags & EFX_PKT_UDP) ? RTE_PTYPE_L4_UDP : 0);
 }
 
+static const uint32_t *
+sfc_efx_supported_ptypes_get(void)
+{
+	static const uint32_t ptypes[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP,
+		RTE_PTYPE_UNKNOWN
+	};
+
+	return ptypes;
+}
+
 static void
-sfc_rx_set_rss_hash(struct sfc_rxq *rxq, unsigned int flags, struct rte_mbuf *m)
+sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags,
+			struct rte_mbuf *m)
 {
 #if EFSYS_OPT_RX_SCALE
 	uint8_t *mbuf_data;
 
 
-	if ((rxq->flags & SFC_RXQ_FLAG_RSS_HASH) == 0)
+	if ((rxq->flags & SFC_EFX_RXQ_FLAG_RSS_HASH) == 0)
 		return;
 
 	mbuf_data = rte_pktmbuf_mtod(m, uint8_t *);
@@ -207,17 +224,18 @@
 #endif
 }
 
-uint16_t
-sfc_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+static uint16_t
+sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
-	struct sfc_rxq *rxq = rx_queue;
+	struct sfc_dp_rxq *dp_rxq = rx_queue;
+	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 	unsigned int completed;
 	unsigned int prefix_size = rxq->prefix_size;
 	unsigned int done_pkts = 0;
 	boolean_t discard_next = B_FALSE;
 	struct rte_mbuf *scatter_pkt = NULL;
 
-	if (unlikely((rxq->flags & SFC_RXQ_FLAG_RUNNING) == 0))
+	if (unlikely((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0))
 		return 0;
 
 	sfc_ev_qpoll(rxq->evq);
@@ -225,7 +243,7 @@
 	completed = rxq->completed;
 	while (completed != rxq->pending && done_pkts < nb_pkts) {
 		unsigned int id;
-		struct sfc_rx_sw_desc *rxd;
+		struct sfc_efx_rx_sw_desc *rxd;
 		struct rte_mbuf *m;
 		unsigned int seg_len;
 		unsigned int desc_flags;
@@ -279,14 +297,16 @@
 		/* The first fragment of the packet has prefix */
 		prefix_size = rxq->prefix_size;
 
-		m->ol_flags = sfc_rx_desc_flags_to_offload_flags(desc_flags);
-		m->packet_type = sfc_rx_desc_flags_to_packet_type(desc_flags);
+		m->ol_flags =
+			sfc_efx_rx_desc_flags_to_offload_flags(desc_flags);
+		m->packet_type =
+			sfc_efx_rx_desc_flags_to_packet_type(desc_flags);
 
 		/*
 		 * Extract RSS hash from the packet prefix and
 		 * set the corresponding field (if needed and possible)
 		 */
-		sfc_rx_set_rss_hash(rxq, desc_flags, m);
+		sfc_efx_rx_set_rss_hash(rxq, desc_flags, m);
 
 		m->data_off += prefix_size;
 
@@ -305,20 +325,18 @@
 
 	rxq->completed = completed;
 
-	sfc_rx_qrefill(rxq);
+	sfc_efx_rx_qrefill(rxq);
 
 	return done_pkts;
 }
 
-unsigned int
-sfc_rx_qdesc_npending(struct sfc_adapter *sa, unsigned int sw_index)
+static sfc_dp_rx_qdesc_npending_t sfc_efx_rx_qdesc_npending;
+static unsigned int
+sfc_efx_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
 {
-	struct sfc_rxq *rxq;
+	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 
-	SFC_ASSERT(sw_index < sa->rxq_count);
-	rxq = sa->rxq_info[sw_index].rxq;
-
-	if (rxq == NULL || (rxq->flags & SFC_RXQ_FLAG_RUNNING) == 0)
+	if ((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0)
 		return 0;
 
 	sfc_ev_qpoll(rxq->evq);
@@ -326,28 +344,170 @@
 	return rxq->pending - rxq->completed;
 }
 
-int
-sfc_rx_qdesc_done(struct sfc_rxq *rxq, unsigned int offset)
+
+static void *
+sfc_efx_rxq_get_ctrl(struct sfc_dp_rxq *dp_rxq)
 {
-	if ((rxq->flags & SFC_RXQ_FLAG_RUNNING) == 0)
-		return 0;
+	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 
-	sfc_ev_qpoll(rxq->evq);
+	return rxq->ctrl;
+}
 
-	return offset < (rxq->pending - rxq->completed);
+static const struct sfc_dp_rxq_ops sfc_efx_ops = {
+	.get_ctrl	= sfc_efx_rxq_get_ctrl,
+};
+
+static sfc_dp_rx_qcreate_t sfc_efx_rx_qcreate;
+static int
+sfc_efx_rx_qcreate(void *ctrl, __rte_unused sfc_dp_exception_t *exception,
+		   int socket_id,
+		   const struct sfc_dp_rx_qcreate_args *args,
+		   struct sfc_dp_rxq **dp_rxqp)
+{
+	struct sfc_efx_rxq *rxq;
+	int rc;
+
+	rc = ENOMEM;
+	rxq = rte_zmalloc_socket("sfc-efx-rxq", sizeof(*rxq),
+				 RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq == NULL)
+		goto fail_rxq_alloc;
+
+	rc = ENOMEM;
+	rxq->sw_desc = rte_calloc_socket("sfc-efx-rxq-sw_desc",
+					 args->rxq_entries,
+					 sizeof(*rxq->sw_desc),
+					 RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq->sw_desc == NULL)
+		goto fail_desc_alloc;
+
+	rxq->ctrl = ctrl;
+	rxq->dp.ops = &sfc_efx_ops;
+
+	rxq->evq = rxq->ctrl->evq;
+	if (args->flags & SFC_RXQ_FLAG_RSS_HASH)
+		rxq->flags |= SFC_EFX_RXQ_FLAG_RSS_HASH;
+	rxq->ptr_mask = args->rxq_entries - 1;
+	rxq->batch_max = args->batch_max;
+	rxq->prefix_size = args->prefix_size;
+	rxq->refill_threshold = args->refill_threshold;
+	rxq->port_id = args->port_id;
+	rxq->buf_size = args->buf_size;
+	rxq->refill_mb_pool = args->refill_mb_pool;
+
+	*dp_rxqp = &rxq->dp;
+	return 0;
+
+fail_desc_alloc:
+	rte_free(rxq);
+
+fail_rxq_alloc:
+	return rc;
+}
+
+static sfc_dp_rx_qdestroy_t sfc_efx_rx_qdestroy;
+static void
+sfc_efx_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
+{
+	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
+
+	rte_free(rxq->sw_desc);
+	rte_free(rxq);
+}
+
+static sfc_dp_rx_qstart_t sfc_efx_rx_qstart;
+static int
+sfc_efx_rx_qstart(struct sfc_dp_rxq *dp_rxq,
+		  __rte_unused unsigned int evq_read_ptr)
+{
+	/* libefx-based datapath is specific to libefx-based PMD */
+	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
+
+	rxq->common = rxq->ctrl->common;
+
+	rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0;
+
+	sfc_efx_rx_qrefill(rxq);
+
+	rxq->flags |= (SFC_EFX_RXQ_FLAG_STARTED | SFC_EFX_RXQ_FLAG_RUNNING);
+
+	return 0;
+}
+
+static sfc_dp_rx_qstop_t sfc_efx_rx_qstop;
+static void
+sfc_efx_rx_qstop(struct sfc_dp_rxq *dp_rxq,
+		 __rte_unused unsigned int *evq_read_ptr)
+{
+	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
+
+	rxq->flags &= ~SFC_EFX_RXQ_FLAG_RUNNING;
+
+	/* libefx-based datapath is bound to libefx-based PMD and uses
+	 * event queue structure directly. So, there is no necessity to
+	 * return EvQ read pointer.
+	 */
 }
 
+static sfc_dp_rx_qpurge_t sfc_efx_rx_qpurge;
 static void
-sfc_rx_qpurge(struct sfc_rxq *rxq)
+sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 {
+	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 	unsigned int i;
-	struct sfc_rx_sw_desc *rxd;
+	struct sfc_efx_rx_sw_desc *rxd;
 
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_desc[i & rxq->ptr_mask];
 		rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
 		rxd->mbuf = NULL;
+		/* Packed stream relies on 0 in inactive SW desc.
+		 * Rx queue stop is not performance critical, so
+		 * there is no harm to do it always.
+		 */
+		rxd->flags = 0;
+		rxd->size = 0;
 	}
+
+	rxq->flags &= ~SFC_EFX_RXQ_FLAG_STARTED;
+}
+
+struct sfc_dp_rx sfc_efx_rx = {
+	.dp = {
+		.name		= SFC_KVARG_DATAPATH_EFX,
+		.type		= SFC_DP_RX,
+		.hw_fw_caps	= 0,
+	},
+	.qcreate		= sfc_efx_rx_qcreate,
+	.qdestroy		= sfc_efx_rx_qdestroy,
+	.qstart			= sfc_efx_rx_qstart,
+	.qstop			= sfc_efx_rx_qstop,
+	.qpurge			= sfc_efx_rx_qpurge,
+	.supported_ptypes_get	= sfc_efx_supported_ptypes_get,
+	.qdesc_npending		= sfc_efx_rx_qdesc_npending,
+	.pkt_burst		= sfc_efx_recv_pkts,
+};
+
+unsigned int
+sfc_rx_qdesc_npending(struct sfc_adapter *sa, unsigned int sw_index)
+{
+	struct sfc_rxq *rxq;
+
+	SFC_ASSERT(sw_index < sa->rxq_count);
+	rxq = sa->rxq_info[sw_index].rxq;
+
+	if (rxq == NULL || (rxq->state & SFC_RXQ_STARTED) == 0)
+		return 0;
+
+	return sa->dp_rx->qdesc_npending(rxq->dp);
+}
+
+int
+sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset)
+{
+	struct sfc_rxq *rxq = dp_rxq->ops->get_ctrl(dp_rxq);
+
+	return offset < rxq->evq->sa->dp_rx->qdesc_npending(dp_rxq);
 }
 
 static void
@@ -398,7 +558,7 @@
 			sfc_info(sa, "RxQ %u flushed", sw_index);
 	}
 
-	sfc_rx_qpurge(rxq);
+	sa->dp_rx->qpurge(rxq->dp);
 }
 
 int
@@ -432,12 +592,11 @@
 
 	efx_rx_qenable(rxq->common);
 
-	rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0;
+	rc = sa->dp_rx->qstart(rxq->dp, evq->read_ptr);
+	if (rc != 0)
+		goto fail_dp_qstart;
 
 	rxq->state |= SFC_RXQ_STARTED;
-	rxq->flags |= SFC_RXQ_FLAG_STARTED | SFC_RXQ_FLAG_RUNNING;
-
-	sfc_rx_qrefill(rxq);
 
 	if (sw_index == 0) {
 		rc = efx_mac_filter_default_rxq_set(sa->nic, rxq->common,
@@ -454,6 +613,9 @@
 	return 0;
 
 fail_mac_filter_default_rxq_set:
+	sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr);
+
+fail_dp_qstart:
 	sfc_rx_qflush(sa, sw_index);
 
 fail_rx_qcreate:
@@ -484,14 +646,13 @@
 	sa->eth_dev->data->rx_queue_state[sw_index] =
 		RTE_ETH_QUEUE_STATE_STOPPED;
 
-	rxq->flags &= ~SFC_RXQ_FLAG_RUNNING;
+	sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr);
 
 	if (sw_index == 0)
 		efx_mac_filter_default_rxq_clear(sa->nic);
 
 	sfc_rx_qflush(sa, sw_index);
 
-	rxq->flags &= ~SFC_RXQ_FLAG_STARTED;
 	rxq->state = SFC_RXQ_INITIALIZED;
 
 	efx_rx_qdestroy(rxq->common);
@@ -629,6 +790,31 @@
 	return buf_size;
 }
 
+static sfc_dp_exception_t sfc_rx_dp_exception;
+static void
+sfc_rx_dp_exception(void *ctrl)
+{
+	struct sfc_rxq *rxq = ctrl;
+	struct sfc_adapter *sa = rxq->evq->sa;
+	unsigned int rxq_sw_index;
+	int rc;
+
+	if (!sfc_adapter_trylock(sa))
+		return;
+
+	rxq_sw_index = sfc_rxq_sw_index(rxq);
+
+	sfc_warn(sa, "restart RxQ %u because of datapath exception",
+		 rxq_sw_index);
+
+	sfc_rx_qstop(sa, rxq_sw_index);
+	rc = sfc_rx_qstart(sa, rxq_sw_index);
+	if (rc != 0)
+		sfc_err(sa, "cannot restart RxQ %u", rxq_sw_index);
+
+	sfc_adapter_unlock(sa);
+}
+
 int
 sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
 	     uint16_t nb_rx_desc, unsigned int socket_id,
@@ -642,6 +828,7 @@
 	unsigned int evq_index;
 	struct sfc_evq *evq;
 	struct sfc_rxq *rxq;
+	struct sfc_dp_rx_qcreate_args args;
 
 	rc = sfc_rx_qcheck_conf(sa, nb_rx_desc, rx_conf);
 	if (rc != 0)
@@ -690,47 +877,51 @@
 	if (rxq == NULL)
 		goto fail_rxq_alloc;
 
-	rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries),
-			   socket_id, &rxq->mem);
-	if (rc != 0)
-		goto fail_dma_alloc;
-
-	rc = ENOMEM;
-	rxq->sw_desc = rte_calloc_socket("sfc-rxq-sw_desc", rxq_info->entries,
-					 sizeof(*rxq->sw_desc),
-					 RTE_CACHE_LINE_SIZE, socket_id);
-	if (rxq->sw_desc == NULL)
-		goto fail_desc_alloc;
+	rxq_info->rxq = rxq;
 
-	evq->rxq = rxq;
 	rxq->evq = evq;
-	rxq->ptr_mask = rxq_info->entries - 1;
+	rxq->hw_index = sw_index;
 	rxq->refill_threshold = rx_conf->rx_free_thresh;
 	rxq->refill_mb_pool = mb_pool;
-	rxq->buf_size = buf_size;
-	rxq->hw_index = sw_index;
-	rxq->port_id = sa->eth_dev->data->port_id;
 
-	/* Cache limits required on datapath in RxQ structure */
-	rxq->batch_max = encp->enc_rx_batch_max;
-	rxq->prefix_size = encp->enc_rx_prefix_size;
+	rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries),
+			   socket_id, &rxq->mem);
+	if (rc != 0)
+		goto fail_dma_alloc;
+
+	memset(&args, 0, sizeof(args));
+	args.refill_mb_pool = rxq->refill_mb_pool;
+	args.refill_threshold = rxq->refill_threshold;
+	args.buf_size = buf_size;
+	args.port_id = sa->eth_dev->data->port_id;
+	args.batch_max = encp->enc_rx_batch_max;
+	args.prefix_size = encp->enc_rx_prefix_size;
 
 #if EFSYS_OPT_RX_SCALE
 	if (sa->hash_support == EFX_RX_HASH_AVAILABLE)
-		rxq->flags |= SFC_RXQ_FLAG_RSS_HASH;
+		args.flags |= SFC_RXQ_FLAG_RSS_HASH;
 #endif
 
+	args.rxq_entries = rxq_info->entries;
+
+	rc = sa->dp_rx->qcreate(rxq, sfc_rx_dp_exception, socket_id, &args,
+				&rxq->dp);
+	if (rc != 0)
+		goto fail_dp_rx_qcreate;
+
+	evq->dp_rxq = rxq->dp;
+
 	rxq->state = SFC_RXQ_INITIALIZED;
 
-	rxq_info->rxq = rxq;
 	rxq_info->deferred_start = (rx_conf->rx_deferred_start != 0);
 
 	return 0;
 
-fail_desc_alloc:
+fail_dp_rx_qcreate:
 	sfc_dma_free(sa, &rxq->mem);
 
 fail_dma_alloc:
+	rxq_info->rxq = NULL;
 	rte_free(rxq);
 
 fail_rxq_alloc:
@@ -757,10 +948,12 @@
 	rxq = rxq_info->rxq;
 	SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED);
 
+	sa->dp_rx->qdestroy(rxq->dp);
+	rxq->dp = NULL;
+
 	rxq_info->rxq = NULL;
 	rxq_info->entries = 0;
 
-	rte_free(rxq->sw_desc);
 	sfc_dma_free(sa, &rxq->mem);
 	rte_free(rxq);
 }
diff --git a/drivers/net/sfc/sfc_rx.h b/drivers/net/sfc/sfc_rx.h
index 881453c..6407406 100644
--- a/drivers/net/sfc/sfc_rx.h
+++ b/drivers/net/sfc/sfc_rx.h
@@ -36,6 +36,8 @@
 
 #include "efx.h"
 
+#include "sfc_dp_rx.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -47,7 +49,7 @@
  * Software Rx descriptor information associated with hardware Rx
  * descriptor.
  */
-struct sfc_rx_sw_desc {
+struct sfc_efx_rx_sw_desc {
 	struct rte_mbuf		*mbuf;
 	unsigned int		flags;
 	unsigned int		size;
@@ -68,35 +70,17 @@ enum sfc_rxq_state_bit {
 };
 
 /**
- * Receive queue information used on data path.
+ * Receive queue control information.
  * Allocated on the socket specified on the queue setup.
  */
 struct sfc_rxq {
-	/* Used on data path */
 	struct sfc_evq		*evq;
-	struct sfc_rx_sw_desc	*sw_desc;
-	unsigned int		flags;
-#define SFC_RXQ_FLAG_STARTED	0x1
-#define SFC_RXQ_FLAG_RUNNING	0x2
-#define SFC_RXQ_FLAG_RSS_HASH	0x4
-	unsigned int		ptr_mask;
-	unsigned int		pending;
-	unsigned int		completed;
-	uint16_t		batch_max;
-	uint16_t		prefix_size;
-
-	/* Used on refill */
-	unsigned int		added;
-	unsigned int		pushed;
-	unsigned int		refill_threshold;
-	uint8_t			port_id;
-	uint16_t		buf_size;
-	struct rte_mempool	*refill_mb_pool;
 	efx_rxq_t		*common;
 	efsys_mem_t		mem;
-
-	/* Not used on data path */
 	unsigned int		hw_index;
+	unsigned int		refill_threshold;
+	struct rte_mempool	*refill_mb_pool;
+	struct sfc_dp_rxq	*dp;
 	unsigned int		state;
 };
 
@@ -113,6 +97,44 @@ struct sfc_rxq {
 }
 
 /**
+ * Receive queue information used on libefx-based data path.
+ * Allocated on the socket specified on the queue setup.
+ */
+struct sfc_efx_rxq {
+	/* Used on data path */
+	struct sfc_evq			*evq;
+	unsigned int			flags;
+#define SFC_EFX_RXQ_FLAG_STARTED	0x1
+#define SFC_EFX_RXQ_FLAG_RUNNING	0x2
+#define SFC_EFX_RXQ_FLAG_RSS_HASH	0x4
+	unsigned int			ptr_mask;
+	unsigned int			pending;
+	unsigned int			completed;
+	uint16_t			batch_max;
+	uint16_t			prefix_size;
+	struct sfc_efx_rx_sw_desc	*sw_desc;
+
+	/* Used on refill */
+	unsigned int			added;
+	unsigned int			pushed;
+	unsigned int			refill_threshold;
+	uint8_t				port_id;
+	uint16_t			buf_size;
+	struct rte_mempool		*refill_mb_pool;
+	efx_rxq_t			*common;
+
+	/* Datapath receive queue anchor */
+	struct sfc_dp_rxq		dp;
+	struct sfc_rxq			*ctrl;
+};
+
+static inline struct sfc_efx_rxq *
+sfc_efx_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq)
+{
+	return container_of(dp_rxq, struct sfc_efx_rxq, dp);
+}
+
+/**
  * Receive queue information used during setup/release only.
  * Allocated on the same socket as adapter data.
  */
@@ -141,12 +163,9 @@ int sfc_rx_qinit(struct sfc_adapter *sa, unsigned int rx_queue_id,
 void sfc_rx_qflush_done(struct sfc_rxq *rxq);
 void sfc_rx_qflush_failed(struct sfc_rxq *rxq);
 
-uint16_t sfc_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
-		       uint16_t nb_pkts);
-
 unsigned int sfc_rx_qdesc_npending(struct sfc_adapter *sa,
 				   unsigned int sw_index);
-int sfc_rx_qdesc_done(struct sfc_rxq *rxq, unsigned int offset);
+int sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset);
 
 #if EFSYS_OPT_RX_SCALE
 efx_rx_hash_type_t sfc_rte_to_efx_hash_type(uint64_t rss_hf);
-- 
1.8.2.3



More information about the dev mailing list