[dpdk-dev] [PATCH 04/13] net/sfc: factor out libefx-based Rx datapath
Andrew Rybchenko
arybchenko at solarflare.com
Thu Mar 2 08:07:10 CET 2017
Split control and datapath to make datapath substitutable and
possibly reusable with alternative control path.
libefx-based Rx datapath is bound to libefx control path, but
other datapaths should be possible to use with alternative
control path(s).
Signed-off-by: Andrew Rybchenko <arybchenko at solarflare.com>
---
doc/guides/nics/sfc_efx.rst | 7 +
drivers/net/sfc/Makefile | 1 +
drivers/net/sfc/sfc.h | 3 +
drivers/net/sfc/sfc_dp.c | 87 ++++++++++++
drivers/net/sfc/sfc_dp.h | 86 ++++++++++++
drivers/net/sfc/sfc_dp_rx.h | 184 +++++++++++++++++++++++++
drivers/net/sfc/sfc_ethdev.c | 105 ++++++++++++---
drivers/net/sfc/sfc_ev.c | 75 ++++++++---
drivers/net/sfc/sfc_ev.h | 4 +-
drivers/net/sfc/sfc_kvargs.c | 10 ++
drivers/net/sfc/sfc_kvargs.h | 8 ++
drivers/net/sfc/sfc_rx.c | 313 ++++++++++++++++++++++++++++++++++---------
drivers/net/sfc/sfc_rx.h | 73 ++++++----
13 files changed, 824 insertions(+), 132 deletions(-)
create mode 100644 drivers/net/sfc/sfc_dp.c
create mode 100644 drivers/net/sfc/sfc_dp.h
create mode 100644 drivers/net/sfc/sfc_dp_rx.h
diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst
index 0a05a0a..8bd8a0c 100644
--- a/doc/guides/nics/sfc_efx.rst
+++ b/doc/guides/nics/sfc_efx.rst
@@ -181,6 +181,13 @@ whitelist option like "-w 02:00.0,arg1=value1,...".
Case-insensitive 1/y/yes/on or 0/n/no/off may be used to specify
boolean parameters value.
+- ``rx_datapath`` [auto|efx] (default **auto**)
+
+ Choose receive datapath implementation.
+ **auto** allows the driver itself to make a choice based on firmware
+ features available and required by the datapath implementation.
+ **efx** chooses libefx-based datapath which supports Rx scatter.
+
- ``perf_profile`` [auto|throughput|low-latency] (default **throughput**)
Choose hardware tunning to be optimized for either throughput or
diff --git a/drivers/net/sfc/Makefile b/drivers/net/sfc/Makefile
index 619a0ed..befff4c 100644
--- a/drivers/net/sfc/Makefile
+++ b/drivers/net/sfc/Makefile
@@ -90,6 +90,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_port.c
SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_rx.c
SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tx.c
SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tso.c
+SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_dp.c
VPATH += $(SRCDIR)/base
diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
index 8c6c02f..2512f2e 100644
--- a/drivers/net/sfc/sfc.h
+++ b/drivers/net/sfc/sfc.h
@@ -136,6 +136,7 @@ struct sfc_intr {
struct sfc_evq_info;
struct sfc_rxq_info;
struct sfc_txq_info;
+struct sfc_dp_rx;
struct sfc_port {
unsigned int lsc_seq;
@@ -209,6 +210,8 @@ struct sfc_adapter {
unsigned int rss_tbl[EFX_RSS_TBL_SIZE];
uint8_t rss_key[SFC_RSS_KEY_SIZE];
#endif
+
+ const struct sfc_dp_rx *dp_rx;
};
/*
diff --git a/drivers/net/sfc/sfc_dp.c b/drivers/net/sfc/sfc_dp.c
new file mode 100644
index 0000000..c4d5fb3
--- /dev/null
+++ b/drivers/net/sfc/sfc_dp.c
@@ -0,0 +1,87 @@
+/*-
+ * Copyright (c) 2017 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/queue.h>
+#include <string.h>
+#include <errno.h>
+
+#include <rte_log.h>
+
+#include "sfc_dp.h"
+
+struct sfc_dp *
+sfc_dp_find_by_name(struct sfc_dp_list *head, enum sfc_dp_type type,
+ const char *name)
+{
+ struct sfc_dp *entry;
+
+ TAILQ_FOREACH(entry, head, links) {
+ if (entry->type != type)
+ continue;
+
+ if (strcmp(entry->name, name) == 0)
+ return entry;
+ }
+
+ return NULL;
+}
+
+struct sfc_dp *
+sfc_dp_find_by_caps(struct sfc_dp_list *head, enum sfc_dp_type type,
+ unsigned int avail_caps)
+{
+ struct sfc_dp *entry;
+
+ TAILQ_FOREACH(entry, head, links) {
+ if (entry->type != type)
+ continue;
+
+ /* Take the first matching */
+ if (sfc_dp_match_hw_fw_caps(entry, avail_caps))
+ return entry;
+ }
+
+ return NULL;
+}
+
+int
+sfc_dp_register(struct sfc_dp_list *head, struct sfc_dp *entry)
+{
+ if (sfc_dp_find_by_name(head, entry->type, entry->name) != NULL) {
+ rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD,
+ "sfc %s dapapath '%s' already registered\n",
+ entry->type == SFC_DP_RX ? "Rx" : "unknown",
+ entry->name);
+ return EEXIST;
+ }
+
+ TAILQ_INSERT_TAIL(head, entry, links);
+
+ return 0;
+}
diff --git a/drivers/net/sfc/sfc_dp.h b/drivers/net/sfc/sfc_dp.h
new file mode 100644
index 0000000..8f78f98
--- /dev/null
+++ b/drivers/net/sfc/sfc_dp.h
@@ -0,0 +1,86 @@
+/*-
+ * Copyright (c) 2017 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SFC_DP_H
+#define _SFC_DP_H
+
+#include <stdbool.h>
+#include <sys/queue.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SFC_DIV_ROUND_UP(a, b) \
+ __extension__ ({ \
+ typeof(a) _a = (a); \
+ typeof(b) _b = (b); \
+ \
+ (_a + (_b - 1)) / _b; \
+ })
+
+/**
+ * Datapath exception handler to be provided by the control path.
+ */
+typedef void (sfc_dp_exception_t)(void *ctrl);
+
+enum sfc_dp_type {
+ SFC_DP_RX = 0, /**< Receive datapath */
+};
+
+/** Datapath definition */
+struct sfc_dp {
+ TAILQ_ENTRY(sfc_dp) links;
+ const char *name;
+ enum sfc_dp_type type;
+ /* Mask of required hardware/firmware capabilities */
+ unsigned int hw_fw_caps;
+};
+
+/** List of datapath variants */
+TAILQ_HEAD(sfc_dp_list, sfc_dp);
+
+/* Check if available HW/FW capabilities are sufficient for the datapath */
+static inline bool
+sfc_dp_match_hw_fw_caps(const struct sfc_dp *dp, unsigned int avail_caps)
+{
+ return (dp->hw_fw_caps & avail_caps) == dp->hw_fw_caps;
+}
+
+struct sfc_dp *sfc_dp_find_by_name(struct sfc_dp_list *head,
+ enum sfc_dp_type type, const char *name);
+struct sfc_dp *sfc_dp_find_by_caps(struct sfc_dp_list *head,
+ enum sfc_dp_type type,
+ unsigned int avail_caps);
+int sfc_dp_register(struct sfc_dp_list *head, struct sfc_dp *entry);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_DP_H */
diff --git a/drivers/net/sfc/sfc_dp_rx.h b/drivers/net/sfc/sfc_dp_rx.h
new file mode 100644
index 0000000..5a714a1
--- /dev/null
+++ b/drivers/net/sfc/sfc_dp_rx.h
@@ -0,0 +1,184 @@
+/*-
+ * Copyright (c) 2017 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SFC_DP_RX_H
+#define _SFC_DP_RX_H
+
+#include <rte_mempool.h>
+#include <rte_ethdev.h>
+
+#include "sfc_dp.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sfc_dp_rxq;
+
+/**
+ * Callback to get control path receive queue by datapath receive queue
+ * handle.
+ */
+typedef void * (sfc_dp_rxq_get_ctrl_t)(struct sfc_dp_rxq *dp_rxq);
+
+/** Datapath receive queue operations */
+struct sfc_dp_rxq_ops {
+ sfc_dp_rxq_get_ctrl_t *get_ctrl;
+};
+
+/**
+ * Generic receive queue information used on data path.
+ * It must be kept as small as it is possible since it is built into
+ * the structure used on datapath.
+ */
+struct sfc_dp_rxq {
+ const struct sfc_dp_rxq_ops *ops;
+};
+
+/**
+ * Datapath receive queue creation arguments.
+ *
+ * The structure is used just to pass information from control path to
+ * datapath. It could be just function arguments, but it would be hardly
+ * readable.
+ */
+struct sfc_dp_rx_qcreate_args {
+ /** Memory pool to allocate Rx buffer from */
+ struct rte_mempool *refill_mb_pool;
+ /** Minimum number of unused Rx descriptors to do refill */
+ unsigned int refill_threshold;
+ /**
+ * Usable mbuf data space in accordance with alignment and
+ * padding requirements imposed by HW.
+ */
+ unsigned int buf_size;
+ /** Port number to be set in the mbuf */
+ uint8_t port_id;
+
+ /**
+ * Maximum number of Rx descriptors completed in one Rx event.
+ * Just for sanity checks if datapath would like to do.
+ */
+ unsigned int batch_max;
+
+ /** Pseudo-header size */
+ unsigned int prefix_size;
+
+ /** Receive queue flags initializer */
+ unsigned int flags;
+#define SFC_RXQ_FLAG_RSS_HASH 0x1
+
+ /** Rx queue size */
+ unsigned int rxq_entries;
+};
+
+/**
+ * Allocate and initalize datapath receive queue.
+ *
+ * @param ctrl Control path Rx queue opaque handle
+ * @param exception Datapath exception handler to bail out to control path
+ * @param socket_id Socket ID to allocate memory
+ * @param args Function arguments wrapped in structure
+ * @param dp_rxqp Location for generic datapath receive queue pointer
+ *
+ * @return 0 or positive errno.
+ */
+typedef int (sfc_dp_rx_qcreate_t)(void *ctrl, sfc_dp_exception_t *exception,
+ int socket_id,
+ const struct sfc_dp_rx_qcreate_args *args,
+ struct sfc_dp_rxq **dp_rxqp);
+
+/**
+ * Free resources allocated for datapath recevie queue.
+ */
+typedef void (sfc_dp_rx_qdestroy_t)(struct sfc_dp_rxq *dp_rxq);
+
+/**
+ * Receive queue start callback.
+ *
+ * It handovers EvQ to the datapath.
+ */
+typedef int (sfc_dp_rx_qstart_t)(struct sfc_dp_rxq *dp_rxq,
+ unsigned int evq_read_ptr);
+
+/**
+ * Receive queue stop function called before flush.
+ */
+typedef void (sfc_dp_rx_qstop_t)(struct sfc_dp_rxq *dp_rxq,
+ unsigned int *evq_read_ptr);
+
+/**
+ * Receive queue purge function called after queue flush.
+ *
+ * Should be used to free unused recevie buffers.
+ */
+typedef void (sfc_dp_rx_qpurge_t)(struct sfc_dp_rxq *dp_rxq);
+
+/** Get packet types recognized/classified */
+typedef const uint32_t * (sfc_dp_rx_supported_ptypes_get_t)(void);
+
+/** Get number of pending Rx descriptors */
+typedef unsigned int (sfc_dp_rx_qdesc_npending_t)(struct sfc_dp_rxq *dp_rxq);
+
+/** Receive datapath definition */
+struct sfc_dp_rx {
+ struct sfc_dp dp;
+
+ sfc_dp_rx_qcreate_t *qcreate;
+ sfc_dp_rx_qdestroy_t *qdestroy;
+ sfc_dp_rx_qstart_t *qstart;
+ sfc_dp_rx_qstop_t *qstop;
+ sfc_dp_rx_qpurge_t *qpurge;
+ sfc_dp_rx_supported_ptypes_get_t *supported_ptypes_get;
+ sfc_dp_rx_qdesc_npending_t *qdesc_npending;
+ eth_rx_burst_t pkt_burst;
+};
+
+static inline struct sfc_dp_rx *
+sfc_dp_find_rx_by_name(struct sfc_dp_list *head, const char *name)
+{
+ struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_RX, name);
+
+ return (p == NULL) ? NULL : container_of(p, struct sfc_dp_rx, dp);
+}
+
+static inline struct sfc_dp_rx *
+sfc_dp_find_rx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps)
+{
+ struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_RX, avail_caps);
+
+ return (p == NULL) ? NULL : container_of(p, struct sfc_dp_rx, dp);
+}
+
+extern struct sfc_dp_rx sfc_efx_rx;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_DP_RX_H */
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index 71587fb..3207cf4 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -30,6 +30,7 @@
#include <rte_dev.h>
#include <rte_ethdev.h>
#include <rte_pci.h>
+#include <rte_errno.h>
#include "efx.h"
@@ -40,7 +41,11 @@
#include "sfc_ev.h"
#include "sfc_rx.h"
#include "sfc_tx.h"
+#include "sfc_dp.h"
+#include "sfc_dp_rx.h"
+static struct sfc_dp_list sfc_dp_head =
+ TAILQ_HEAD_INITIALIZER(sfc_dp_head);
static void
sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
@@ -114,19 +119,9 @@
static const uint32_t *
sfc_dev_supported_ptypes_get(struct rte_eth_dev *dev)
{
- static const uint32_t ptypes[] = {
- RTE_PTYPE_L2_ETHER,
- RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
- RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
- RTE_PTYPE_L4_TCP,
- RTE_PTYPE_L4_UDP,
- RTE_PTYPE_UNKNOWN
- };
-
- if (dev->rx_pkt_burst == sfc_recv_pkts)
- return ptypes;
-
- return NULL;
+ struct sfc_adapter *sa = dev->data->dev_private;
+
+ return sa->dp_rx->supported_ptypes_get();
}
static int
@@ -366,7 +361,7 @@
if (rc != 0)
goto fail_rx_qinit;
- dev->data->rx_queues[rx_queue_id] = sa->rxq_info[rx_queue_id].rxq;
+ dev->data->rx_queues[rx_queue_id] = sa->rxq_info[rx_queue_id].rxq->dp;
sfc_adapter_unlock(sa);
@@ -381,13 +376,15 @@
static void
sfc_rx_queue_release(void *queue)
{
- struct sfc_rxq *rxq = queue;
+ struct sfc_dp_rxq *dp_rxq = queue;
+ struct sfc_rxq *rxq;
struct sfc_adapter *sa;
unsigned int sw_index;
- if (rxq == NULL)
+ if (dp_rxq == NULL)
return;
+ rxq = dp_rxq->ops->get_ctrl(dp_rxq);
sa = rxq->evq->sa;
sfc_adapter_lock(sa);
@@ -905,9 +902,9 @@
static int
sfc_rx_descriptor_done(void *queue, uint16_t offset)
{
- struct sfc_rxq *rxq = queue;
+ struct sfc_dp_rxq *dp_rxq = queue;
- return sfc_rx_qdesc_done(rxq, offset);
+ return sfc_rx_qdesc_done(dp_rxq, offset);
}
static int
@@ -1230,6 +1227,69 @@
};
static int
+sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
+{
+ struct sfc_adapter *sa = dev->data->dev_private;
+ unsigned int avail_caps = 0;
+ const char *rx_name = NULL;
+ int rc;
+
+ if (sa == NULL || sa->state == SFC_ADAPTER_UNINITIALIZED)
+ return -E_RTE_SECONDARY;
+
+ rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH,
+ sfc_kvarg_string_handler, &rx_name);
+ if (rc != 0)
+ goto fail_kvarg_rx_datapath;
+
+ if (rx_name != NULL) {
+ sa->dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, rx_name);
+ if (sa->dp_rx == NULL) {
+ sfc_err(sa, "Rx datapath %s not found", rx_name);
+ rc = ENOENT;
+ goto fail_dp_rx;
+ }
+ if (!sfc_dp_match_hw_fw_caps(&sa->dp_rx->dp, avail_caps)) {
+ sfc_err(sa,
+ "Insufficient Hw/FW capabilities to use Rx datapath %s",
+ rx_name);
+ rc = EINVAL;
+ goto fail_dp_rx;
+ }
+ } else {
+ sa->dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps);
+ if (sa->dp_rx == NULL) {
+ sfc_err(sa, "Rx datapath by caps %#x not found",
+ avail_caps);
+ rc = ENOENT;
+ goto fail_dp_rx;
+ }
+ }
+
+ sfc_info(sa, "use %s Rx datapath", sa->dp_rx->dp.name);
+
+ dev->rx_pkt_burst = sa->dp_rx->pkt_burst;
+
+ dev->tx_pkt_burst = sfc_xmit_pkts;
+
+ dev->dev_ops = &sfc_eth_dev_ops;
+
+ return 0;
+
+fail_dp_rx:
+fail_kvarg_rx_datapath:
+ return rc;
+}
+
+static void
+sfc_register_dp(void)
+{
+ /* Register once */
+ if (TAILQ_EMPTY(&sfc_dp_head))
+ sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
+}
+
+static int
sfc_eth_dev_init(struct rte_eth_dev *dev)
{
struct sfc_adapter *sa = dev->data->dev_private;
@@ -1238,6 +1298,8 @@
const efx_nic_cfg_t *encp;
const struct ether_addr *from;
+ sfc_register_dp();
+
/* Required for logging */
sa->eth_dev = dev;
@@ -1278,12 +1340,10 @@
from = (const struct ether_addr *)(encp->enc_mac_addr);
ether_addr_copy(from, &dev->data->mac_addrs[0]);
- dev->dev_ops = &sfc_eth_dev_ops;
- dev->rx_pkt_burst = &sfc_recv_pkts;
- dev->tx_pkt_burst = &sfc_xmit_pkts;
-
sfc_adapter_unlock(sa);
+ sfc_eth_dev_set_ops(dev);
+
sfc_log_init(sa, "done");
return 0;
@@ -1358,6 +1418,7 @@
RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map);
RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio");
RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
+ SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " "
SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " "
SFC_KVARG_MCDI_LOGGING "=" SFC_KVARG_VALUES_BOOL " "
SFC_KVARG_DEBUG_INIT "=" SFC_KVARG_VALUES_BOOL);
diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c
index 7f79c7e..dca3a81 100644
--- a/drivers/net/sfc/sfc_ev.c
+++ b/drivers/net/sfc/sfc_ev.c
@@ -81,25 +81,25 @@
}
static boolean_t
-sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
- uint32_t size, uint16_t flags)
+sfc_ev_efx_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
+ uint32_t size, uint16_t flags)
{
struct sfc_evq *evq = arg;
- struct sfc_rxq *rxq;
+ struct sfc_efx_rxq *rxq;
unsigned int stop;
unsigned int pending_id;
unsigned int delta;
unsigned int i;
- struct sfc_rx_sw_desc *rxd;
+ struct sfc_efx_rx_sw_desc *rxd;
if (unlikely(evq->exception))
goto done;
- rxq = evq->rxq;
+ rxq = sfc_efx_rxq_by_dp_rxq(evq->dp_rxq);
SFC_ASSERT(rxq != NULL);
SFC_ASSERT(rxq->evq == evq);
- SFC_ASSERT(rxq->flags & SFC_RXQ_FLAG_STARTED);
+ SFC_ASSERT(rxq->flags & SFC_EFX_RXQ_FLAG_STARTED);
stop = (id + 1) & rxq->ptr_mask;
pending_id = rxq->pending & rxq->ptr_mask;
@@ -117,7 +117,9 @@
sfc_err(evq->sa,
"EVQ %u RxQ %u invalid RX abort "
"(id=%#x size=%u flags=%#x); needs restart",
- evq->evq_index, sfc_rxq_sw_index(rxq),
+ evq->evq_index,
+ sfc_rxq_sw_index_by_hw_index(
+ rxq->ctrl->hw_index),
id, size, flags);
goto done;
}
@@ -132,8 +134,9 @@
sfc_err(evq->sa,
"EVQ %u RxQ %u completion out of order "
"(id=%#x delta=%u flags=%#x); needs restart",
- evq->evq_index, sfc_rxq_sw_index(rxq), id, delta,
- flags);
+ evq->evq_index,
+ sfc_rxq_sw_index_by_hw_index(rxq->ctrl->hw_index),
+ id, delta, flags);
goto done;
}
@@ -231,9 +234,13 @@
sfc_ev_rxq_flush_done(void *arg, __rte_unused uint32_t rxq_hw_index)
{
struct sfc_evq *evq = arg;
+ struct sfc_dp_rxq *dp_rxq;
struct sfc_rxq *rxq;
- rxq = evq->rxq;
+ dp_rxq = evq->dp_rxq;
+ SFC_ASSERT(dp_rxq != NULL);
+
+ rxq = dp_rxq->ops->get_ctrl(dp_rxq);
SFC_ASSERT(rxq != NULL);
SFC_ASSERT(rxq->hw_index == rxq_hw_index);
SFC_ASSERT(rxq->evq == evq);
@@ -256,9 +263,13 @@
sfc_ev_rxq_flush_failed(void *arg, __rte_unused uint32_t rxq_hw_index)
{
struct sfc_evq *evq = arg;
+ struct sfc_dp_rxq *dp_rxq;
struct sfc_rxq *rxq;
- rxq = evq->rxq;
+ dp_rxq = evq->dp_rxq;
+ SFC_ASSERT(dp_rxq != NULL);
+
+ rxq = dp_rxq->ops->get_ctrl(dp_rxq);
SFC_ASSERT(rxq != NULL);
SFC_ASSERT(rxq->hw_index == rxq_hw_index);
SFC_ASSERT(rxq->evq == evq);
@@ -387,9 +398,24 @@
.eec_link_change = sfc_ev_link_change,
};
-static const efx_ev_callbacks_t sfc_ev_callbacks_rx = {
+static const efx_ev_callbacks_t sfc_ev_callbacks_efx_rx = {
.eec_initialized = sfc_ev_initialized,
- .eec_rx = sfc_ev_rx,
+ .eec_rx = sfc_ev_efx_rx,
+ .eec_tx = sfc_ev_nop_tx,
+ .eec_exception = sfc_ev_exception,
+ .eec_rxq_flush_done = sfc_ev_rxq_flush_done,
+ .eec_rxq_flush_failed = sfc_ev_rxq_flush_failed,
+ .eec_txq_flush_done = sfc_ev_nop_txq_flush_done,
+ .eec_software = sfc_ev_software,
+ .eec_sram = sfc_ev_sram,
+ .eec_wake_up = sfc_ev_wake_up,
+ .eec_timer = sfc_ev_timer,
+ .eec_link_change = sfc_ev_nop_link_change,
+};
+
+static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx = {
+ .eec_initialized = sfc_ev_initialized,
+ .eec_rx = sfc_ev_nop_rx,
.eec_tx = sfc_ev_nop_tx,
.eec_exception = sfc_ev_exception,
.eec_rxq_flush_done = sfc_ev_rxq_flush_done,
@@ -432,9 +458,12 @@
struct sfc_adapter *sa = evq->sa;
int rc;
- if ((evq->rxq != NULL) &&
- (evq->rxq->flags & SFC_RXQ_FLAG_RUNNING)) {
- unsigned int rxq_sw_index = sfc_rxq_sw_index(evq->rxq);
+ if (evq->dp_rxq != NULL) {
+ struct sfc_rxq *rxq;
+ unsigned int rxq_sw_index;
+
+ rxq = evq->dp_rxq->ops->get_ctrl(evq->dp_rxq);
+ rxq_sw_index = sfc_rxq_sw_index(rxq);
sfc_warn(sa,
"restart RxQ %u because of exception on its EvQ %u",
@@ -518,13 +547,17 @@
if (rc != 0)
goto fail_ev_qcreate;
- SFC_ASSERT(evq->rxq == NULL || evq->txq == NULL);
- if (evq->rxq != 0)
- evq->callbacks = &sfc_ev_callbacks_rx;
- else if (evq->txq != 0)
+ SFC_ASSERT(evq->dp_rxq == NULL || evq->txq == NULL);
+ if (evq->dp_rxq != 0) {
+ if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0)
+ evq->callbacks = &sfc_ev_callbacks_efx_rx;
+ else
+ evq->callbacks = &sfc_ev_callbacks_dp_rx;
+ } else if (evq->txq != 0) {
evq->callbacks = &sfc_ev_callbacks_tx;
- else
+ } else {
evq->callbacks = &sfc_ev_callbacks;
+ }
evq->init_state = SFC_EVQ_STARTING;
diff --git a/drivers/net/sfc/sfc_ev.h b/drivers/net/sfc/sfc_ev.h
index 41a37f4..e99cd74 100644
--- a/drivers/net/sfc/sfc_ev.h
+++ b/drivers/net/sfc/sfc_ev.h
@@ -40,7 +40,7 @@
#define SFC_MGMT_EVQ_ENTRIES (EFX_EVQ_MINNEVS)
struct sfc_adapter;
-struct sfc_rxq;
+struct sfc_dp_rxq;
struct sfc_txq;
enum sfc_evq_state {
@@ -59,7 +59,7 @@ struct sfc_evq {
unsigned int read_ptr;
boolean_t exception;
efsys_mem_t mem;
- struct sfc_rxq *rxq;
+ struct sfc_dp_rxq *dp_rxq;
struct sfc_txq *txq;
/* Not used on datapath */
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 227a8db..d8529fa 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -45,6 +45,7 @@
SFC_KVARG_DEBUG_INIT,
SFC_KVARG_MCDI_LOGGING,
SFC_KVARG_PERF_PROFILE,
+ SFC_KVARG_RX_DATAPATH,
NULL,
};
@@ -110,3 +111,12 @@
return 0;
}
+
+int
+sfc_kvarg_string_handler(__rte_unused const char *key,
+ const char *value_str, void *opaque)
+{
+ *(const char **)opaque = value_str;
+
+ return 0;
+}
diff --git a/drivers/net/sfc/sfc_kvargs.h b/drivers/net/sfc/sfc_kvargs.h
index 2fea9c7..2d0ffde 100644
--- a/drivers/net/sfc/sfc_kvargs.h
+++ b/drivers/net/sfc/sfc_kvargs.h
@@ -52,6 +52,12 @@
SFC_KVARG_PERF_PROFILE_THROUGHPUT "|" \
SFC_KVARG_PERF_PROFILE_LOW_LATENCY "]"
+#define SFC_KVARG_DATAPATH_EFX "efx"
+
+#define SFC_KVARG_RX_DATAPATH "rx_datapath"
+#define SFC_KVARG_VALUES_RX_DATAPATH \
+ "[" SFC_KVARG_DATAPATH_EFX "]"
+
struct sfc_adapter;
int sfc_kvargs_parse(struct sfc_adapter *sa);
@@ -62,6 +68,8 @@ int sfc_kvargs_process(struct sfc_adapter *sa, const char *key_match,
int sfc_kvarg_bool_handler(const char *key, const char *value_str,
void *opaque);
+int sfc_kvarg_string_handler(const char *key, const char *value_str,
+ void *opaque);
#ifdef __cplusplus
}
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index 61654fc..e56837d 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -36,6 +36,7 @@
#include "sfc_log.h"
#include "sfc_ev.h"
#include "sfc_rx.h"
+#include "sfc_kvargs.h"
#include "sfc_tweak.h"
/*
@@ -72,7 +73,7 @@
}
static void
-sfc_rx_qrefill(struct sfc_rxq *rxq)
+sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq)
{
unsigned int free_space;
unsigned int bulks;
@@ -81,7 +82,7 @@
unsigned int added = rxq->added;
unsigned int id;
unsigned int i;
- struct sfc_rx_sw_desc *rxd;
+ struct sfc_efx_rx_sw_desc *rxd;
struct rte_mbuf *m;
uint8_t port_id = rxq->port_id;
@@ -135,7 +136,7 @@
}
static uint64_t
-sfc_rx_desc_flags_to_offload_flags(const unsigned int desc_flags)
+sfc_efx_rx_desc_flags_to_offload_flags(const unsigned int desc_flags)
{
uint64_t mbuf_flags = 0;
@@ -174,7 +175,7 @@
}
static uint32_t
-sfc_rx_desc_flags_to_packet_type(const unsigned int desc_flags)
+sfc_efx_rx_desc_flags_to_packet_type(const unsigned int desc_flags)
{
return RTE_PTYPE_L2_ETHER |
((desc_flags & EFX_PKT_IPV4) ?
@@ -185,14 +186,30 @@
((desc_flags & EFX_PKT_UDP) ? RTE_PTYPE_L4_UDP : 0);
}
+static const uint32_t *
+sfc_efx_supported_ptypes_get(void)
+{
+ static const uint32_t ptypes[] = {
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_UNKNOWN
+ };
+
+ return ptypes;
+}
+
static void
-sfc_rx_set_rss_hash(struct sfc_rxq *rxq, unsigned int flags, struct rte_mbuf *m)
+sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags,
+ struct rte_mbuf *m)
{
#if EFSYS_OPT_RX_SCALE
uint8_t *mbuf_data;
- if ((rxq->flags & SFC_RXQ_FLAG_RSS_HASH) == 0)
+ if ((rxq->flags & SFC_EFX_RXQ_FLAG_RSS_HASH) == 0)
return;
mbuf_data = rte_pktmbuf_mtod(m, uint8_t *);
@@ -207,17 +224,18 @@
#endif
}
-uint16_t
-sfc_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+static uint16_t
+sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
- struct sfc_rxq *rxq = rx_queue;
+ struct sfc_dp_rxq *dp_rxq = rx_queue;
+ struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
unsigned int completed;
unsigned int prefix_size = rxq->prefix_size;
unsigned int done_pkts = 0;
boolean_t discard_next = B_FALSE;
struct rte_mbuf *scatter_pkt = NULL;
- if (unlikely((rxq->flags & SFC_RXQ_FLAG_RUNNING) == 0))
+ if (unlikely((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0))
return 0;
sfc_ev_qpoll(rxq->evq);
@@ -225,7 +243,7 @@
completed = rxq->completed;
while (completed != rxq->pending && done_pkts < nb_pkts) {
unsigned int id;
- struct sfc_rx_sw_desc *rxd;
+ struct sfc_efx_rx_sw_desc *rxd;
struct rte_mbuf *m;
unsigned int seg_len;
unsigned int desc_flags;
@@ -279,14 +297,16 @@
/* The first fragment of the packet has prefix */
prefix_size = rxq->prefix_size;
- m->ol_flags = sfc_rx_desc_flags_to_offload_flags(desc_flags);
- m->packet_type = sfc_rx_desc_flags_to_packet_type(desc_flags);
+ m->ol_flags =
+ sfc_efx_rx_desc_flags_to_offload_flags(desc_flags);
+ m->packet_type =
+ sfc_efx_rx_desc_flags_to_packet_type(desc_flags);
/*
* Extract RSS hash from the packet prefix and
* set the corresponding field (if needed and possible)
*/
- sfc_rx_set_rss_hash(rxq, desc_flags, m);
+ sfc_efx_rx_set_rss_hash(rxq, desc_flags, m);
m->data_off += prefix_size;
@@ -305,20 +325,18 @@
rxq->completed = completed;
- sfc_rx_qrefill(rxq);
+ sfc_efx_rx_qrefill(rxq);
return done_pkts;
}
-unsigned int
-sfc_rx_qdesc_npending(struct sfc_adapter *sa, unsigned int sw_index)
+static sfc_dp_rx_qdesc_npending_t sfc_efx_rx_qdesc_npending;
+static unsigned int
+sfc_efx_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
{
- struct sfc_rxq *rxq;
+ struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
- SFC_ASSERT(sw_index < sa->rxq_count);
- rxq = sa->rxq_info[sw_index].rxq;
-
- if (rxq == NULL || (rxq->flags & SFC_RXQ_FLAG_RUNNING) == 0)
+ if ((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0)
return 0;
sfc_ev_qpoll(rxq->evq);
@@ -326,28 +344,170 @@
return rxq->pending - rxq->completed;
}
-int
-sfc_rx_qdesc_done(struct sfc_rxq *rxq, unsigned int offset)
+
+static void *
+sfc_efx_rxq_get_ctrl(struct sfc_dp_rxq *dp_rxq)
{
- if ((rxq->flags & SFC_RXQ_FLAG_RUNNING) == 0)
- return 0;
+ struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
- sfc_ev_qpoll(rxq->evq);
+ return rxq->ctrl;
+}
- return offset < (rxq->pending - rxq->completed);
+static const struct sfc_dp_rxq_ops sfc_efx_ops = {
+ .get_ctrl = sfc_efx_rxq_get_ctrl,
+};
+
+static sfc_dp_rx_qcreate_t sfc_efx_rx_qcreate;
+static int
+sfc_efx_rx_qcreate(void *ctrl, __rte_unused sfc_dp_exception_t *exception,
+ int socket_id,
+ const struct sfc_dp_rx_qcreate_args *args,
+ struct sfc_dp_rxq **dp_rxqp)
+{
+ struct sfc_efx_rxq *rxq;
+ int rc;
+
+ rc = ENOMEM;
+ rxq = rte_zmalloc_socket("sfc-efx-rxq", sizeof(*rxq),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (rxq == NULL)
+ goto fail_rxq_alloc;
+
+ rc = ENOMEM;
+ rxq->sw_desc = rte_calloc_socket("sfc-efx-rxq-sw_desc",
+ args->rxq_entries,
+ sizeof(*rxq->sw_desc),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (rxq->sw_desc == NULL)
+ goto fail_desc_alloc;
+
+ rxq->ctrl = ctrl;
+ rxq->dp.ops = &sfc_efx_ops;
+
+ rxq->evq = rxq->ctrl->evq;
+ if (args->flags & SFC_RXQ_FLAG_RSS_HASH)
+ rxq->flags |= SFC_EFX_RXQ_FLAG_RSS_HASH;
+ rxq->ptr_mask = args->rxq_entries - 1;
+ rxq->batch_max = args->batch_max;
+ rxq->prefix_size = args->prefix_size;
+ rxq->refill_threshold = args->refill_threshold;
+ rxq->port_id = args->port_id;
+ rxq->buf_size = args->buf_size;
+ rxq->refill_mb_pool = args->refill_mb_pool;
+
+ *dp_rxqp = &rxq->dp;
+ return 0;
+
+fail_desc_alloc:
+ rte_free(rxq);
+
+fail_rxq_alloc:
+ return rc;
+}
+
+static sfc_dp_rx_qdestroy_t sfc_efx_rx_qdestroy;
+static void
+sfc_efx_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
+{
+ struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
+
+ rte_free(rxq->sw_desc);
+ rte_free(rxq);
+}
+
+static sfc_dp_rx_qstart_t sfc_efx_rx_qstart;
+static int
+sfc_efx_rx_qstart(struct sfc_dp_rxq *dp_rxq,
+ __rte_unused unsigned int evq_read_ptr)
+{
+ /* libefx-based datapath is specific to libefx-based PMD */
+ struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
+
+ rxq->common = rxq->ctrl->common;
+
+ rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0;
+
+ sfc_efx_rx_qrefill(rxq);
+
+ rxq->flags |= (SFC_EFX_RXQ_FLAG_STARTED | SFC_EFX_RXQ_FLAG_RUNNING);
+
+ return 0;
+}
+
+static sfc_dp_rx_qstop_t sfc_efx_rx_qstop;
+static void
+sfc_efx_rx_qstop(struct sfc_dp_rxq *dp_rxq,
+ __rte_unused unsigned int *evq_read_ptr)
+{
+ struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
+
+ rxq->flags &= ~SFC_EFX_RXQ_FLAG_RUNNING;
+
+ /* libefx-based datapath is bound to libefx-based PMD and uses
+ * event queue structure directly. So, there is no necessity to
+ * return EvQ read pointer.
+ */
}
+static sfc_dp_rx_qpurge_t sfc_efx_rx_qpurge;
static void
-sfc_rx_qpurge(struct sfc_rxq *rxq)
+sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
{
+ struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
unsigned int i;
- struct sfc_rx_sw_desc *rxd;
+ struct sfc_efx_rx_sw_desc *rxd;
for (i = rxq->completed; i != rxq->added; ++i) {
rxd = &rxq->sw_desc[i & rxq->ptr_mask];
rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
rxd->mbuf = NULL;
+ /* Packed stream relies on 0 in inactive SW desc.
+ * Rx queue stop is not performance critical, so
+ * there is no harm to do it always.
+ */
+ rxd->flags = 0;
+ rxd->size = 0;
}
+
+ rxq->flags &= ~SFC_EFX_RXQ_FLAG_STARTED;
+}
+
+struct sfc_dp_rx sfc_efx_rx = {
+ .dp = {
+ .name = SFC_KVARG_DATAPATH_EFX,
+ .type = SFC_DP_RX,
+ .hw_fw_caps = 0,
+ },
+ .qcreate = sfc_efx_rx_qcreate,
+ .qdestroy = sfc_efx_rx_qdestroy,
+ .qstart = sfc_efx_rx_qstart,
+ .qstop = sfc_efx_rx_qstop,
+ .qpurge = sfc_efx_rx_qpurge,
+ .supported_ptypes_get = sfc_efx_supported_ptypes_get,
+ .qdesc_npending = sfc_efx_rx_qdesc_npending,
+ .pkt_burst = sfc_efx_recv_pkts,
+};
+
+unsigned int
+sfc_rx_qdesc_npending(struct sfc_adapter *sa, unsigned int sw_index)
+{
+ struct sfc_rxq *rxq;
+
+ SFC_ASSERT(sw_index < sa->rxq_count);
+ rxq = sa->rxq_info[sw_index].rxq;
+
+ if (rxq == NULL || (rxq->state & SFC_RXQ_STARTED) == 0)
+ return 0;
+
+ return sa->dp_rx->qdesc_npending(rxq->dp);
+}
+
+int
+sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset)
+{
+ struct sfc_rxq *rxq = dp_rxq->ops->get_ctrl(dp_rxq);
+
+ return offset < rxq->evq->sa->dp_rx->qdesc_npending(dp_rxq);
}
static void
@@ -398,7 +558,7 @@
sfc_info(sa, "RxQ %u flushed", sw_index);
}
- sfc_rx_qpurge(rxq);
+ sa->dp_rx->qpurge(rxq->dp);
}
int
@@ -432,12 +592,11 @@
efx_rx_qenable(rxq->common);
- rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0;
+ rc = sa->dp_rx->qstart(rxq->dp, evq->read_ptr);
+ if (rc != 0)
+ goto fail_dp_qstart;
rxq->state |= SFC_RXQ_STARTED;
- rxq->flags |= SFC_RXQ_FLAG_STARTED | SFC_RXQ_FLAG_RUNNING;
-
- sfc_rx_qrefill(rxq);
if (sw_index == 0) {
rc = efx_mac_filter_default_rxq_set(sa->nic, rxq->common,
@@ -454,6 +613,9 @@
return 0;
fail_mac_filter_default_rxq_set:
+ sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr);
+
+fail_dp_qstart:
sfc_rx_qflush(sa, sw_index);
fail_rx_qcreate:
@@ -484,14 +646,13 @@
sa->eth_dev->data->rx_queue_state[sw_index] =
RTE_ETH_QUEUE_STATE_STOPPED;
- rxq->flags &= ~SFC_RXQ_FLAG_RUNNING;
+ sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr);
if (sw_index == 0)
efx_mac_filter_default_rxq_clear(sa->nic);
sfc_rx_qflush(sa, sw_index);
- rxq->flags &= ~SFC_RXQ_FLAG_STARTED;
rxq->state = SFC_RXQ_INITIALIZED;
efx_rx_qdestroy(rxq->common);
@@ -629,6 +790,31 @@
return buf_size;
}
+static sfc_dp_exception_t sfc_rx_dp_exception;
+static void
+sfc_rx_dp_exception(void *ctrl)
+{
+ struct sfc_rxq *rxq = ctrl;
+ struct sfc_adapter *sa = rxq->evq->sa;
+ unsigned int rxq_sw_index;
+ int rc;
+
+ if (!sfc_adapter_trylock(sa))
+ return;
+
+ rxq_sw_index = sfc_rxq_sw_index(rxq);
+
+ sfc_warn(sa, "restart RxQ %u because of datapath exception",
+ rxq_sw_index);
+
+ sfc_rx_qstop(sa, rxq_sw_index);
+ rc = sfc_rx_qstart(sa, rxq_sw_index);
+ if (rc != 0)
+ sfc_err(sa, "cannot restart RxQ %u", rxq_sw_index);
+
+ sfc_adapter_unlock(sa);
+}
+
int
sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
uint16_t nb_rx_desc, unsigned int socket_id,
@@ -642,6 +828,7 @@
unsigned int evq_index;
struct sfc_evq *evq;
struct sfc_rxq *rxq;
+ struct sfc_dp_rx_qcreate_args args;
rc = sfc_rx_qcheck_conf(sa, nb_rx_desc, rx_conf);
if (rc != 0)
@@ -690,47 +877,51 @@
if (rxq == NULL)
goto fail_rxq_alloc;
- rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries),
- socket_id, &rxq->mem);
- if (rc != 0)
- goto fail_dma_alloc;
-
- rc = ENOMEM;
- rxq->sw_desc = rte_calloc_socket("sfc-rxq-sw_desc", rxq_info->entries,
- sizeof(*rxq->sw_desc),
- RTE_CACHE_LINE_SIZE, socket_id);
- if (rxq->sw_desc == NULL)
- goto fail_desc_alloc;
+ rxq_info->rxq = rxq;
- evq->rxq = rxq;
rxq->evq = evq;
- rxq->ptr_mask = rxq_info->entries - 1;
+ rxq->hw_index = sw_index;
rxq->refill_threshold = rx_conf->rx_free_thresh;
rxq->refill_mb_pool = mb_pool;
- rxq->buf_size = buf_size;
- rxq->hw_index = sw_index;
- rxq->port_id = sa->eth_dev->data->port_id;
- /* Cache limits required on datapath in RxQ structure */
- rxq->batch_max = encp->enc_rx_batch_max;
- rxq->prefix_size = encp->enc_rx_prefix_size;
+ rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries),
+ socket_id, &rxq->mem);
+ if (rc != 0)
+ goto fail_dma_alloc;
+
+ memset(&args, 0, sizeof(args));
+ args.refill_mb_pool = rxq->refill_mb_pool;
+ args.refill_threshold = rxq->refill_threshold;
+ args.buf_size = buf_size;
+ args.port_id = sa->eth_dev->data->port_id;
+ args.batch_max = encp->enc_rx_batch_max;
+ args.prefix_size = encp->enc_rx_prefix_size;
#if EFSYS_OPT_RX_SCALE
if (sa->hash_support == EFX_RX_HASH_AVAILABLE)
- rxq->flags |= SFC_RXQ_FLAG_RSS_HASH;
+ args.flags |= SFC_RXQ_FLAG_RSS_HASH;
#endif
+ args.rxq_entries = rxq_info->entries;
+
+ rc = sa->dp_rx->qcreate(rxq, sfc_rx_dp_exception, socket_id, &args,
+ &rxq->dp);
+ if (rc != 0)
+ goto fail_dp_rx_qcreate;
+
+ evq->dp_rxq = rxq->dp;
+
rxq->state = SFC_RXQ_INITIALIZED;
- rxq_info->rxq = rxq;
rxq_info->deferred_start = (rx_conf->rx_deferred_start != 0);
return 0;
-fail_desc_alloc:
+fail_dp_rx_qcreate:
sfc_dma_free(sa, &rxq->mem);
fail_dma_alloc:
+ rxq_info->rxq = NULL;
rte_free(rxq);
fail_rxq_alloc:
@@ -757,10 +948,12 @@
rxq = rxq_info->rxq;
SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED);
+ sa->dp_rx->qdestroy(rxq->dp);
+ rxq->dp = NULL;
+
rxq_info->rxq = NULL;
rxq_info->entries = 0;
- rte_free(rxq->sw_desc);
sfc_dma_free(sa, &rxq->mem);
rte_free(rxq);
}
diff --git a/drivers/net/sfc/sfc_rx.h b/drivers/net/sfc/sfc_rx.h
index 881453c..6407406 100644
--- a/drivers/net/sfc/sfc_rx.h
+++ b/drivers/net/sfc/sfc_rx.h
@@ -36,6 +36,8 @@
#include "efx.h"
+#include "sfc_dp_rx.h"
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -47,7 +49,7 @@
* Software Rx descriptor information associated with hardware Rx
* descriptor.
*/
-struct sfc_rx_sw_desc {
+struct sfc_efx_rx_sw_desc {
struct rte_mbuf *mbuf;
unsigned int flags;
unsigned int size;
@@ -68,35 +70,17 @@ enum sfc_rxq_state_bit {
};
/**
- * Receive queue information used on data path.
+ * Receive queue control information.
* Allocated on the socket specified on the queue setup.
*/
struct sfc_rxq {
- /* Used on data path */
struct sfc_evq *evq;
- struct sfc_rx_sw_desc *sw_desc;
- unsigned int flags;
-#define SFC_RXQ_FLAG_STARTED 0x1
-#define SFC_RXQ_FLAG_RUNNING 0x2
-#define SFC_RXQ_FLAG_RSS_HASH 0x4
- unsigned int ptr_mask;
- unsigned int pending;
- unsigned int completed;
- uint16_t batch_max;
- uint16_t prefix_size;
-
- /* Used on refill */
- unsigned int added;
- unsigned int pushed;
- unsigned int refill_threshold;
- uint8_t port_id;
- uint16_t buf_size;
- struct rte_mempool *refill_mb_pool;
efx_rxq_t *common;
efsys_mem_t mem;
-
- /* Not used on data path */
unsigned int hw_index;
+ unsigned int refill_threshold;
+ struct rte_mempool *refill_mb_pool;
+ struct sfc_dp_rxq *dp;
unsigned int state;
};
@@ -113,6 +97,44 @@ struct sfc_rxq {
}
/**
+ * Receive queue information used on libefx-based data path.
+ * Allocated on the socket specified on the queue setup.
+ */
+struct sfc_efx_rxq {
+ /* Used on data path */
+ struct sfc_evq *evq;
+ unsigned int flags;
+#define SFC_EFX_RXQ_FLAG_STARTED 0x1
+#define SFC_EFX_RXQ_FLAG_RUNNING 0x2
+#define SFC_EFX_RXQ_FLAG_RSS_HASH 0x4
+ unsigned int ptr_mask;
+ unsigned int pending;
+ unsigned int completed;
+ uint16_t batch_max;
+ uint16_t prefix_size;
+ struct sfc_efx_rx_sw_desc *sw_desc;
+
+ /* Used on refill */
+ unsigned int added;
+ unsigned int pushed;
+ unsigned int refill_threshold;
+ uint8_t port_id;
+ uint16_t buf_size;
+ struct rte_mempool *refill_mb_pool;
+ efx_rxq_t *common;
+
+ /* Datapath receive queue anchor */
+ struct sfc_dp_rxq dp;
+ struct sfc_rxq *ctrl;
+};
+
+static inline struct sfc_efx_rxq *
+sfc_efx_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq)
+{
+ return container_of(dp_rxq, struct sfc_efx_rxq, dp);
+}
+
+/**
* Receive queue information used during setup/release only.
* Allocated on the same socket as adapter data.
*/
@@ -141,12 +163,9 @@ int sfc_rx_qinit(struct sfc_adapter *sa, unsigned int rx_queue_id,
void sfc_rx_qflush_done(struct sfc_rxq *rxq);
void sfc_rx_qflush_failed(struct sfc_rxq *rxq);
-uint16_t sfc_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts);
-
unsigned int sfc_rx_qdesc_npending(struct sfc_adapter *sa,
unsigned int sw_index);
-int sfc_rx_qdesc_done(struct sfc_rxq *rxq, unsigned int offset);
+int sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset);
#if EFSYS_OPT_RX_SCALE
efx_rx_hash_type_t sfc_rte_to_efx_hash_type(uint64_t rss_hf);
--
1.8.2.3
More information about the dev
mailing list