[dpdk-dev] [PATCH 13/13] net/sfc: support Rx packed stream EF10-specific datapath

Andrew Rybchenko arybchenko at solarflare.com
Thu Mar 2 08:07:19 CET 2017


Signed-off-by: Andrew Rybchenko <arybchenko at solarflare.com>
---
 doc/guides/nics/sfc_efx.rst      |  23 +-
 drivers/net/sfc/Makefile         |   1 +
 drivers/net/sfc/efsys.h          |   2 +-
 drivers/net/sfc/sfc_dp.h         |   5 +-
 drivers/net/sfc/sfc_dp_rx.h      |   8 +
 drivers/net/sfc/sfc_ef10_ps_rx.c | 659 +++++++++++++++++++++++++++++++++++++++
 drivers/net/sfc/sfc_ethdev.c     |   7 +
 drivers/net/sfc/sfc_ev.c         |  34 ++
 drivers/net/sfc/sfc_kvargs.h     |   4 +-
 drivers/net/sfc/sfc_rx.c         |   5 +-
 drivers/net/sfc/sfc_rx.h         |   2 +-
 11 files changed, 744 insertions(+), 6 deletions(-)
 create mode 100644 drivers/net/sfc/sfc_ef10_ps_rx.c

diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst
index 8e2b36f..d52ab45 100644
--- a/doc/guides/nics/sfc_efx.rst
+++ b/doc/guides/nics/sfc_efx.rst
@@ -114,6 +114,24 @@ required in the receive buffer.
 It should be taken into account when mbuf pool for receive is created.
 
 
+Packed stream mode
+~~~~~~~~~~~~~~~~~~
+
+When the receive queue is in packed stream mode, the driver handles mbufs
+differently.
+The mbufs that are handed over to the application are always indirect mbufs.
+The contents of mbufs may be freely modified by the application, with an
+important constraint: the indirect mbufs does not have any reserved "head room"
+before the actual contents (or rather it is used by PMD internally),
+so the usual practice of prepending an extra header through manipulating
+the head room **will not** work and would result in corrupted packets.
+If one needs to prepend data to a packet, one should first create a copy of
+mbuf.
+
+Another limitation of a packed stream mode, imposed by the firmware, is that
+it allows for a single RSS context.
+
+
 Supported NICs
 --------------
 
@@ -181,7 +199,7 @@ whitelist option like "-w 02:00.0,arg1=value1,...".
 Case-insensitive 1/y/yes/on or 0/n/no/off may be used to specify
 boolean parameters value.
 
-- ``rx_datapath`` [auto|efx|ef10] (default **auto**)
+- ``rx_datapath`` [auto|efx|ef10|ef10_packed] (default **auto**)
 
   Choose receive datapath implementation.
   **auto** allows the driver itself to make a choice based on firmware
@@ -190,6 +208,9 @@ boolean parameters value.
   **ef10** chooses EF10 (SFN7xxx, SFN8xxx) native datapath which is
   more efficient than libefx-based and provides richer packet type
   classification, but lacks Rx scatter support.
+  **ef10_packed** chooses EF10 (SFN7xxx, SFN8xxx) packed stream datapath
+  which may be used on capture packed stream firmware variant only
+  (see notes about its limitations above).
 
 - ``tx_datapath`` [auto|efx|ef10|ef10_simple] (default **auto**)
 
diff --git a/drivers/net/sfc/Makefile b/drivers/net/sfc/Makefile
index bb7dcb2..dd83238 100644
--- a/drivers/net/sfc/Makefile
+++ b/drivers/net/sfc/Makefile
@@ -92,6 +92,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tx.c
 SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tso.c
 SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_dp.c
 SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ef10_rx.c
+SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ef10_ps_rx.c
 SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ef10_tx.c
 
 VPATH += $(SRCDIR)/base
diff --git a/drivers/net/sfc/efsys.h b/drivers/net/sfc/efsys.h
index 60829be..98e591c 100644
--- a/drivers/net/sfc/efsys.h
+++ b/drivers/net/sfc/efsys.h
@@ -210,7 +210,7 @@
 
 #define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0
 
-#define EFSYS_OPT_RX_PACKED_STREAM 0
+#define EFSYS_OPT_RX_PACKED_STREAM 1
 
 /* ID */
 
diff --git a/drivers/net/sfc/sfc_dp.h b/drivers/net/sfc/sfc_dp.h
index d3e7007..362a998 100644
--- a/drivers/net/sfc/sfc_dp.h
+++ b/drivers/net/sfc/sfc_dp.h
@@ -37,6 +37,8 @@
 extern "C" {
 #endif
 
+#define SFC_P2_ROUND_UP(x, align)	(-(-(x) & -(align)))
+
 #define SFC_DIV_ROUND_UP(a, b) \
 	__extension__ ({		\
 		typeof(a) _a = (a);	\
@@ -62,7 +64,8 @@ struct sfc_dp {
 	enum sfc_dp_type		type;
 	/* Mask of required hardware/firmware capabilities */
 	unsigned int			hw_fw_caps;
-#define SFC_DP_HW_FW_CAP_EF10		0x1
+#define SFC_DP_HW_FW_CAP_EF10				0x1
+#define SFC_DP_HW_FW_CAP_RX_PACKED_STREAM_64K		0x2
 };
 
 /** List of datapath variants */
diff --git a/drivers/net/sfc/sfc_dp_rx.h b/drivers/net/sfc/sfc_dp_rx.h
index 944d366..0541c43 100644
--- a/drivers/net/sfc/sfc_dp_rx.h
+++ b/drivers/net/sfc/sfc_dp_rx.h
@@ -154,6 +154,12 @@ typedef void (sfc_dp_rx_qstop_t)(struct sfc_dp_rxq *dp_rxq,
 typedef bool (sfc_dp_rx_qrx_ev_t)(struct sfc_dp_rxq *dp_rxq, unsigned int id);
 
 /**
+ * Packed stream receive event handler used during queue flush only.
+ */
+typedef bool (sfc_dp_rx_qrx_ps_ev_t)(struct sfc_dp_rxq *dp_rxq,
+				     unsigned int id);
+
+/**
  * Receive queue purge function called after queue flush.
  *
  * Should be used to free unused recevie buffers.
@@ -177,6 +183,7 @@ struct sfc_dp_rx {
 	sfc_dp_rx_qstart_t			*qstart;
 	sfc_dp_rx_qstop_t			*qstop;
 	sfc_dp_rx_qrx_ev_t			*qrx_ev;
+	sfc_dp_rx_qrx_ps_ev_t			*qrx_ps_ev;
 	sfc_dp_rx_qpurge_t			*qpurge;
 	sfc_dp_rx_supported_ptypes_get_t	*supported_ptypes_get;
 	sfc_dp_rx_qdesc_npending_t		*qdesc_npending;
@@ -201,6 +208,7 @@ struct sfc_dp_rx {
 
 extern struct sfc_dp_rx sfc_efx_rx;
 extern struct sfc_dp_rx sfc_ef10_rx;
+extern struct sfc_dp_rx sfc_ef10_ps_rx;
 
 #ifdef __cplusplus
 }
diff --git a/drivers/net/sfc/sfc_ef10_ps_rx.c b/drivers/net/sfc/sfc_ef10_ps_rx.c
new file mode 100644
index 0000000..6365092
--- /dev/null
+++ b/drivers/net/sfc/sfc_ef10_ps_rx.c
@@ -0,0 +1,659 @@
+/*-
+ * Copyright (c) 2017 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* EF10 packed stream native datapath implementation */
+
+#include <stdbool.h>
+
+#include <rte_byteorder.h>
+#include <rte_mbuf_ptype.h>
+#include <rte_mbuf.h>
+#include <rte_io.h>
+
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_regs_ef10.h"
+
+#include "sfc_tweak.h"
+#include "sfc_dp_rx.h"
+#include "sfc_kvargs.h"
+
+#if 1
+/* Alignment requirement for value written to RX WPTR:
+ *  the WPTR must be aligned to an 8 descriptor boundary
+ */
+#define	EF10_RX_WPTR_ALIGN 8
+#define	EFX_RX_PACKED_STREAM_ALIGNMENT 64
+#define	EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE 8
+#endif
+
+#define SFC_PACKED_STREAM_BUFSIZE (64 * 1024)
+
+
+struct sfc_ef10_ps_rx_sw_desc {
+	struct rte_mbuf			*mbuf;
+};
+
+struct sfc_ef10_ps_rxq {
+	/* Used on data path */
+	unsigned int			flags;
+#define SFC_EF10_PS_RXQ_STARTED		0x1
+#define SFC_EF10_PS_RXQ_RUNNING		0x2
+#define SFC_EF10_PS_RXQ_EXCEPTION	0x4
+	unsigned int			rxq_ptr_mask;
+	unsigned int			completed;
+	unsigned int			pending_pkts;
+	const uint8_t			*next_pkt;
+	unsigned int			packets;
+	unsigned int			evq_read_ptr;
+	unsigned int			evq_ptr_mask;
+	volatile efx_qword_t		*evq_hw_ring;
+	struct sfc_ef10_ps_rx_sw_desc	*sw_ring;
+	struct rte_mempool		*indirect_mb_pool;
+	uint8_t				port_id;
+	uint8_t				credits;
+
+	/* Used on refill */
+	unsigned int			added;
+	unsigned int			refill_threshold;
+	struct rte_mempool		*refill_mb_pool;
+	efx_qword_t			*rxq_hw_ring;
+	volatile void			*doorbell;
+
+	/* Datapath receive queue anchor */
+	struct sfc_dp_rxq		dp;
+	void				*ctrl;
+	sfc_dp_exception_t		*exception;
+};
+
+static inline struct sfc_ef10_ps_rxq *
+sfc_ef10_ps_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq)
+{
+	return container_of(dp_rxq, struct sfc_ef10_ps_rxq, dp);
+}
+
+static void
+sfc_ef10_ps_rx_qpush(struct sfc_ef10_ps_rxq *rxq)
+{
+	efx_dword_t dword;
+
+	/* Hardware has alignment restriction for WPTR */
+	RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % EF10_RX_WPTR_ALIGN != 0);
+	SFC_ASSERT(RTE_ALIGN(rxq->added, EF10_RX_WPTR_ALIGN) == rxq->added);
+
+	EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR,
+			     rxq->added & rxq->rxq_ptr_mask);
+
+	/* Make sure that all descriptor update (Rx and event) reach memory */
+	rte_wmb();
+
+	/* DMA sync to device is not required */
+
+	rte_write32(dword.ed_u32[0], rxq->doorbell);
+}
+
+static void
+sfc_ef10_ps_rx_update_credits(struct sfc_ef10_ps_rxq *rxq)
+{
+	efx_dword_t dword;
+
+	if (rxq->credits == 0)
+		return;
+
+	EFX_POPULATE_DWORD_3(dword,
+	    ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
+	    ERF_DZ_RX_DESC_MAGIC_CMD,
+	    ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
+	    ERF_DZ_RX_DESC_MAGIC_DATA, rxq->credits);
+
+	/* Make sure that event descriptor update reach memory */
+	rte_wmb();
+
+	/* DMA sync to device is not required */
+
+	rte_write32(dword.ed_u32[0], rxq->doorbell);
+
+	rxq->credits = 0;
+}
+
+static void
+sfc_ef10_ps_rx_qrefill(struct sfc_ef10_ps_rxq *rxq)
+{
+	const unsigned int rxq_ptr_mask = rxq->rxq_ptr_mask;
+	unsigned int free_space;
+	unsigned int bulks;
+	void *objs[SFC_RX_REFILL_BULK];
+	unsigned int added = rxq->added;
+
+	free_space = EFX_RXQ_LIMIT(rxq_ptr_mask + 1) - (added - rxq->completed);
+
+	if (free_space < rxq->refill_threshold)
+		return;
+
+	bulks = free_space / RTE_DIM(objs);
+
+	while (bulks-- > 0) {
+		unsigned int id;
+		unsigned int i;
+
+		if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs,
+						  RTE_DIM(objs)) < 0)) {
+			struct rte_eth_dev_data *dev_data =
+				rte_eth_devices[rxq->port_id].data;
+
+			/*
+			 * It is hardly a safe way to increment counter
+			 * from different contexts, but all PMDs do it.
+			 */
+			dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs);
+			break;
+		}
+
+		for (i = 0, id = added & rxq_ptr_mask;
+		     i < RTE_DIM(objs);
+		     ++i, ++id) {
+			struct rte_mbuf *m = objs[i];
+			struct sfc_ef10_ps_rx_sw_desc *rxd;
+			unsigned int req_align;
+			uintptr_t dp;
+			uintptr_t adjust_align;
+
+			SFC_ASSERT((id & ~rxq_ptr_mask) == 0);
+			rxd = &rxq->sw_ring[id];
+			rxd->mbuf = m;
+
+			rte_mbuf_refcnt_set(m, 1);
+
+			req_align = SFC_PACKED_STREAM_BUFSIZE;
+			dp = rte_pktmbuf_mtophys(m) + req_align;
+			adjust_align = RTE_ALIGN_CEIL(dp, req_align) - dp;
+
+			/*
+			 * Align using priv_size to be able to
+			 * find correct address of the mbuf on
+			 * detach (see rte_mbuf_from_indirect()).
+			 */
+			m->buf_addr = RTE_PTR_ADD(m->buf_addr, adjust_align);
+			m->buf_physaddr += adjust_align;
+			m->priv_size += adjust_align;
+
+			EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id],
+			    ESF_DZ_RX_KER_BYTE_CNT,
+			    EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE,
+			    ESF_DZ_RX_KER_BUF_ADDR, m->buf_physaddr);
+		}
+
+		added += RTE_DIM(objs);
+	}
+
+	/* Push doorbell if something is posted */
+	if (likely(rxq->added != added)) {
+		rxq->added = added;
+		sfc_ef10_ps_rx_qpush(rxq);
+	}
+}
+
+static uint16_t
+sfc_ef10_ps_rx_get_pending(struct sfc_ef10_ps_rxq *rxq,
+			   struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending_pkts);
+	struct rte_mbuf *hb;
+	const uint8_t *next_pkt;
+	unsigned int i;
+
+	if (n_rx_pkts == 0)
+		return 0;
+
+	rxq->pending_pkts -= n_rx_pkts;
+
+	if (rte_mempool_get_bulk(rxq->indirect_mb_pool,
+				 (void **)rx_pkts,
+				 n_rx_pkts) < 0)
+		return 0;
+
+	hb = rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask].mbuf;
+	next_pkt = rxq->next_pkt;
+
+	for (i = 0; i < n_rx_pkts; ++i) {
+		struct rte_mbuf *m = rx_pkts[i];
+		const efx_qword_t *qwordp;
+		uint16_t pkt_len;
+		uint16_t buf_len;
+
+		/* Parse pseudo-header */
+		qwordp = (const efx_qword_t *)next_pkt;
+		pkt_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
+		buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
+
+		/* Prepare indirect mbuf */
+		rte_mbuf_refcnt_set(m, 1);
+		/* reference counter is incremented on huge mbuf */
+		rte_pktmbuf_attach(m, hb);
+		m->data_off = next_pkt - (uint8_t *)hb->buf_addr +
+			      EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
+		rte_pktmbuf_pkt_len(m) = pkt_len;
+		rte_pktmbuf_data_len(m) = pkt_len;
+		m->packet_type = RTE_PTYPE_L2_ETHER;
+
+		/* Move to the next packet */
+		buf_len = SFC_P2_ROUND_UP(buf_len +
+					  EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
+					  EFX_RX_PACKED_STREAM_ALIGNMENT);
+		next_pkt += buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
+	}
+
+	rxq->next_pkt = next_pkt;
+
+	return n_rx_pkts;
+}
+
+static void
+sfc_ef10_ps_rx_discard_pending(struct sfc_ef10_ps_rxq *rxq)
+{
+	const uint8_t *next_pkt;
+	unsigned int i;
+
+	printf("DISCARD\n");
+
+	next_pkt = rxq->next_pkt;
+
+	for (i = 0; i < rxq->pending_pkts; ++i) {
+		const efx_qword_t *qwordp;
+		uint16_t buf_len;
+
+		qwordp = (const efx_qword_t *)next_pkt;
+		buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
+		buf_len = SFC_P2_ROUND_UP(buf_len +
+					  EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
+					  EFX_RX_PACKED_STREAM_ALIGNMENT);
+		next_pkt += buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
+	}
+
+	rxq->next_pkt = next_pkt;
+	rxq->pending_pkts = 0;
+}
+
+static void
+sfc_ef10_ps_rx_process_ev(struct sfc_ef10_ps_rxq *rxq, efx_qword_t rx_ev)
+{
+	unsigned int ready;
+
+	SFC_ASSERT(rxq->pending_pkts == 0);
+
+	ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) -
+		 rxq->packets) &
+		EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
+
+	rxq->packets += ready;
+	rxq->pending_pkts = ready;
+
+	if (EFX_TEST_QWORD_BIT(rx_ev, ESF_DZ_RX_EV_ROTATE_LBN)) {
+		struct sfc_ef10_ps_rx_sw_desc *rxd;
+
+		/* Credit is spent by firmware */
+		rxq->credits++;
+
+		/* Drop our reference to huge buffer */
+		rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
+		rte_pktmbuf_free(rxd->mbuf);
+
+		/* Switch to the next huge buffer */
+		rxq->completed++;
+		rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
+		rxq->next_pkt = rxd->mbuf->buf_addr;
+	}
+
+	if (rx_ev.eq_u64[0] &
+	    rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) |
+			     (1ull << ESF_DZ_RX_ECRC_ERR_LBN)))
+		sfc_ef10_ps_rx_discard_pending(rxq);
+}
+
+static bool
+sfc_ef10_ps_rx_event_get(struct sfc_ef10_ps_rxq *rxq, efx_qword_t *rx_ev)
+{
+	if (unlikely(rxq->flags & SFC_EF10_PS_RXQ_EXCEPTION))
+		return false;
+
+	*rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->evq_ptr_mask];
+
+	if (rx_ev->eq_u64[0] == UINT64_MAX)
+		return false;
+
+	if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) !=
+		     FSE_AZ_EV_CODE_RX_EV)) {
+		/*
+		 * Do not move read_ptr to keep the event for exception
+		 * handling
+		 */
+		rxq->flags |= SFC_EF10_PS_RXQ_EXCEPTION;
+		return false;
+	}
+
+	rxq->evq_read_ptr++;
+	return true;
+}
+
+static void
+sfc_ef10_ps_ev_qfill(struct sfc_ef10_ps_rxq *rxq, unsigned int old_read_ptr)
+{
+	const unsigned int read_ptr = rxq->evq_read_ptr;
+	const unsigned int evq_ptr_mask = rxq->evq_ptr_mask;
+
+	while (old_read_ptr != read_ptr) {
+		EFX_SET_QWORD(rxq->evq_hw_ring[old_read_ptr & evq_ptr_mask]);
+		++old_read_ptr;
+	}
+
+	/*
+	 * No barriers here.
+	 * Functions which push doorbell should care about correct
+	 * ordering: store instructions which fill in EvQ ring should be
+	 * retired from CPU and DMA sync before doorbell which will allow
+	 * to use these event entries.
+	 */
+}
+
+static uint16_t
+sfc_ef10_ps_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+		      uint16_t nb_pkts)
+{
+	struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(rx_queue);
+	const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
+	uint16_t n_rx_pkts;
+	efx_qword_t rx_ev;
+
+	if (unlikely((rxq->flags & SFC_EF10_PS_RXQ_RUNNING) == 0))
+		return 0;
+
+	n_rx_pkts = sfc_ef10_ps_rx_get_pending(rxq, rx_pkts, nb_pkts);
+
+	while (n_rx_pkts != nb_pkts && sfc_ef10_ps_rx_event_get(rxq, &rx_ev)) {
+		if (EFX_TEST_QWORD_BIT(rx_ev, ESF_DZ_RX_DROP_EVENT_LBN))
+			continue;
+
+		sfc_ef10_ps_rx_process_ev(rxq, rx_ev);
+		n_rx_pkts += sfc_ef10_ps_rx_get_pending(rxq,
+							rx_pkts + n_rx_pkts,
+							nb_pkts - n_rx_pkts);
+	}
+
+	sfc_ef10_ps_ev_qfill(rxq, evq_old_read_ptr);
+
+	if (unlikely(rxq->flags & SFC_EF10_PS_RXQ_EXCEPTION)) {
+		/* Exception handling may restart the RxQ */
+		/* So, make sure that no cached variables are used after */
+		rxq->exception(rxq->ctrl);
+	} else {
+		sfc_ef10_ps_rx_update_credits(rxq);
+		sfc_ef10_ps_rx_qrefill(rxq);
+	}
+
+	return n_rx_pkts;
+}
+
+static const uint32_t *
+sfc_ef10_ps_supported_ptypes_get(void)
+{
+	static const uint32_t ef10_packed_ptypes[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_UNKNOWN
+	};
+
+	return ef10_packed_ptypes;
+}
+
+static sfc_dp_rx_qdesc_npending_t sfc_ef10_ps_rx_qdesc_npending;
+static unsigned int
+sfc_ef10_ps_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq)
+{
+	/*
+	 * Correct implementation requires EvQ polling and events
+	 * processing.
+	 */
+	return -ENOTSUP;
+}
+
+static void *
+sfc_ef10_ps_rxq_get_ctrl(struct sfc_dp_rxq *dp_rxq)
+{
+	struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+
+	return rxq->ctrl;
+}
+
+static const struct sfc_dp_rxq_ops sfc_ef10_ps_rxq_ops = {
+	.get_ctrl	= sfc_ef10_ps_rxq_get_ctrl,
+};
+
+
+static struct rte_mempool *
+sfc_ef10_ps_rx_huge_pktmbuf_pool_create(struct sfc_ef10_ps_rxq *rxq,
+					unsigned int hw_index, int socket_id)
+{
+	struct rte_pktmbuf_pool_private mbp_priv;
+	unsigned int elt_size;
+	char hb_pool_name[64];
+
+	/* Twice size to guarantee alignment */
+	elt_size = sizeof(struct rte_mbuf) + SFC_PACKED_STREAM_BUFSIZE * 2;
+
+	/* mbufs from this pool are not real mbufs, they are
+	 * used solely to hold large data buffers, so most mbuf-specific
+	 * fields are really unimportant. In fact mbuf data room size
+	 * does not fit into 16-bit integer.
+	 */
+	memset(&mbp_priv, 0, sizeof(mbp_priv));
+
+	snprintf(hb_pool_name, sizeof(hb_pool_name),
+		 "sfc-hugebuf%u.%u", rxq->port_id, hw_index);
+
+	/* ptr_mask is the number of entres in the queue minus 1,
+	 * which happens to be the optimal size for rte_mempool_create
+	 */
+	return rte_mempool_create(hb_pool_name, rxq->rxq_ptr_mask, elt_size,
+				  0, sizeof(struct rte_pktmbuf_pool_private),
+				  rte_pktmbuf_pool_init, &mbp_priv,
+				  rte_pktmbuf_init, NULL,
+				  socket_id, 0);
+}
+
+static sfc_dp_rx_qcreate_t sfc_ef10_ps_rx_qcreate;
+static int
+sfc_ef10_ps_rx_qcreate(void *ctrl, sfc_dp_exception_t *exception,
+		       int socket_id,
+		       const struct sfc_dp_rx_qcreate_args *args,
+		       struct sfc_dp_rxq **dp_rxqp)
+{
+	struct sfc_ef10_ps_rxq *rxq;
+	int rc;
+
+	rc = ENOMEM;
+	rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq),
+				 RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq == NULL)
+		goto fail_rxq_alloc;
+
+	rc = ENOMEM;
+	rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring",
+					 args->rxq_entries,
+					 sizeof(*rxq->sw_ring),
+					 RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq->sw_ring == NULL)
+		goto fail_desc_alloc;
+
+	rxq->rxq_ptr_mask = args->rxq_entries - 1;
+	rxq->evq_ptr_mask = args->evq_entries - 1;
+	rxq->evq_hw_ring = args->evq_hw_ring;
+	rxq->refill_threshold = args->refill_threshold;
+	rxq->port_id = args->port_id;
+	rxq->indirect_mb_pool = args->refill_mb_pool;
+	rxq->rxq_hw_ring = args->rxq_hw_ring;
+
+	rc = ENOMEM;
+	rxq->refill_mb_pool =
+		sfc_ef10_ps_rx_huge_pktmbuf_pool_create(rxq, args->hw_index,
+							socket_id);
+	if (rxq->refill_mb_pool == NULL)
+		goto fail_huge_pktmbuf_pool_create;
+
+	rxq->doorbell = (volatile uint8_t *)args->mem_bar +
+			ER_DZ_RX_DESC_UPD_REG_OFST +
+			args->hw_index * ER_DZ_RX_DESC_UPD_REG_STEP;
+
+	rxq->dp.ops = &sfc_ef10_ps_rxq_ops;
+	rxq->ctrl = ctrl;
+	rxq->exception = exception;
+
+	*dp_rxqp = &rxq->dp;
+	return 0;
+
+fail_huge_pktmbuf_pool_create:
+	rte_free(rxq->sw_ring);
+
+fail_desc_alloc:
+	rte_free(rxq);
+
+fail_rxq_alloc:
+	return rc;
+}
+
+static sfc_dp_rx_qdestroy_t sfc_ef10_ps_rx_qdestroy;
+static void
+sfc_ef10_ps_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
+{
+	struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+
+	rte_mempool_free(rxq->refill_mb_pool);
+	rte_free(rxq->sw_ring);
+	rte_free(rxq);
+}
+
+static sfc_dp_rx_qstart_t sfc_ef10_ps_rx_qstart;
+static int
+sfc_ef10_ps_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr)
+{
+	struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+	struct sfc_ef10_ps_rx_sw_desc *rxd;
+
+	rxq->pending_pkts = 0;
+	rxq->evq_read_ptr = evq_read_ptr;
+
+	/* Initialize before refill */
+	rxq->completed = rxq->added = 0;
+
+	sfc_ef10_ps_rx_qrefill(rxq);
+
+	/* Step back to handle the first EV_ROTATE correctly */
+	rxq->completed--;
+	/*
+	 * Allocate dummy mbuf to be freed on the first EV_ROTATE.
+	 * It is not used, so do not bother to initialize it.
+	 */
+	rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
+	rxd->mbuf = rte_mbuf_raw_alloc(rxq->refill_mb_pool);
+	if (rxd->mbuf == NULL)
+		return ENOMEM;
+
+	rxq->flags |= (SFC_EF10_PS_RXQ_STARTED | SFC_EF10_PS_RXQ_RUNNING);
+	rxq->flags &= ~SFC_EF10_PS_RXQ_EXCEPTION;
+
+	/*
+	 * Control path grants initial packed stream credits to firmware
+	 * in accordance with event queue size. We simply track when
+	 * credits are spent and refill.
+	 */
+
+	return 0;
+}
+
+static sfc_dp_rx_qstop_t sfc_ef10_ps_rx_qstop;
+static void
+sfc_ef10_ps_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr)
+{
+	struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+
+	rxq->flags &= ~SFC_EF10_PS_RXQ_RUNNING;
+
+	*evq_read_ptr = rxq->evq_read_ptr;
+}
+
+static sfc_dp_rx_qrx_ev_t sfc_ef10_ps_rx_qrx_ev;
+static bool
+sfc_ef10_ps_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id)
+{
+	__rte_unused struct sfc_ef10_ps_rxq *rxq;
+
+	rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+	SFC_ASSERT(~rxq->flags & SFC_EF10_PS_RXQ_RUNNING);
+
+	/*
+	 * It is safe to ignore Rx event since we free all mbufs on
+	 * queue purge anyway.
+	 */
+
+	return false;
+}
+
+static sfc_dp_rx_qpurge_t sfc_ef10_ps_rx_qpurge;
+static void
+sfc_ef10_ps_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
+{
+	struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+	unsigned int i;
+	struct sfc_ef10_ps_rx_sw_desc *rxd;
+
+	for (i = rxq->completed; i != rxq->added; ++i) {
+		rxd = &rxq->sw_ring[i & rxq->rxq_ptr_mask];
+		rte_pktmbuf_free(rxd->mbuf);
+	}
+
+	rxq->flags &= ~SFC_EF10_PS_RXQ_STARTED;
+}
+
+struct sfc_dp_rx sfc_ef10_ps_rx = {
+	.dp = {
+		.name		= SFC_KVARG_DATAPATH_EF10_RX_PACKED,
+		.type		= SFC_DP_RX,
+		.hw_fw_caps	= SFC_DP_HW_FW_CAP_EF10 |
+				  SFC_DP_HW_FW_CAP_RX_PACKED_STREAM_64K,
+	},
+	.features		= 0,
+	.qcreate		= sfc_ef10_ps_rx_qcreate,
+	.qdestroy		= sfc_ef10_ps_rx_qdestroy,
+	.qstart			= sfc_ef10_ps_rx_qstart,
+	.qstop			= sfc_ef10_ps_rx_qstop,
+	.qrx_ev			= sfc_ef10_ps_rx_qrx_ev,
+	.qpurge			= sfc_ef10_ps_rx_qpurge,
+	.supported_ptypes_get	= sfc_ef10_ps_supported_ptypes_get,
+	.qdesc_npending		= sfc_ef10_ps_rx_qdesc_npending,
+	.pkt_burst		= sfc_ef10_ps_recv_pkts,
+};
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index ffa6abe..e36f18b 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -1234,6 +1234,7 @@
 sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
 {
 	struct sfc_adapter *sa = dev->data->dev_private;
+	const efx_nic_cfg_t *encp;
 	unsigned int avail_caps = 0;
 	const char *rx_name = NULL;
 	const char *tx_name = NULL;
@@ -1251,6 +1252,11 @@
 		break;
 	}
 
+	encp = efx_nic_cfg_get(sa->nic);
+	/* 64k buffers can be supported only */
+	if (encp->enc_rx_var_packed_stream_supported)
+		avail_caps |= SFC_DP_HW_FW_CAP_RX_PACKED_STREAM_64K;
+
 	rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH,
 				sfc_kvarg_string_handler, &rx_name);
 	if (rc != 0)
@@ -1334,6 +1340,7 @@
 	/* Register once */
 	if (TAILQ_EMPTY(&sfc_dp_head)) {
 		/* Prefer EF10 datapath */
+		sfc_dp_register(&sfc_dp_head, &sfc_ef10_ps_rx.dp);
 		sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp);
 		sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
 
diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c
index 5e7c619..c26af3e 100644
--- a/drivers/net/sfc/sfc_ev.c
+++ b/drivers/net/sfc/sfc_ev.c
@@ -171,6 +171,35 @@
 }
 
 static boolean_t
+sfc_ev_nop_rx_ps(void *arg, uint32_t label, uint32_t id,
+		 uint32_t pkt_count, uint16_t flags)
+{
+	struct sfc_evq *evq = arg;
+
+	sfc_err(evq->sa,
+		"EVQ %u unexpected packed stream Rx event label=%u id=%#x pkt_count=%u flags=%#x",
+		evq->evq_index, label, id, pkt_count, flags);
+	return B_TRUE;
+}
+
+/* It is not actually used on datapath, but required on RxQ flush */
+static boolean_t
+sfc_ev_dp_rx_ps(void *arg, __rte_unused uint32_t label, uint32_t id,
+		__rte_unused uint32_t pkt_count, __rte_unused uint16_t flags)
+{
+	struct sfc_evq *evq = arg;
+	struct sfc_dp_rxq *dp_rxq;
+
+	dp_rxq = evq->dp_rxq;
+	SFC_ASSERT(dp_rxq != NULL);
+
+	if (evq->sa->dp_rx->qrx_ps_ev != NULL)
+		return evq->sa->dp_rx->qrx_ps_ev(dp_rxq, id);
+	else
+		return B_FALSE;
+}
+
+static boolean_t
 sfc_ev_nop_tx(void *arg, uint32_t label, uint32_t id)
 {
 	struct sfc_evq *evq = arg;
@@ -419,6 +448,7 @@
 static const efx_ev_callbacks_t sfc_ev_callbacks = {
 	.eec_initialized	= sfc_ev_initialized,
 	.eec_rx			= sfc_ev_nop_rx,
+	.eec_rx_ps		= sfc_ev_nop_rx_ps,
 	.eec_tx			= sfc_ev_nop_tx,
 	.eec_exception		= sfc_ev_exception,
 	.eec_rxq_flush_done	= sfc_ev_nop_rxq_flush_done,
@@ -434,6 +464,7 @@
 static const efx_ev_callbacks_t sfc_ev_callbacks_efx_rx = {
 	.eec_initialized	= sfc_ev_initialized,
 	.eec_rx			= sfc_ev_efx_rx,
+	.eec_rx_ps		= sfc_ev_nop_rx_ps,
 	.eec_tx			= sfc_ev_nop_tx,
 	.eec_exception		= sfc_ev_exception,
 	.eec_rxq_flush_done	= sfc_ev_rxq_flush_done,
@@ -449,6 +480,7 @@
 static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx = {
 	.eec_initialized	= sfc_ev_initialized,
 	.eec_rx			= sfc_ev_dp_rx,
+	.eec_rx_ps		= sfc_ev_dp_rx_ps,
 	.eec_tx			= sfc_ev_nop_tx,
 	.eec_exception		= sfc_ev_exception,
 	.eec_rxq_flush_done	= sfc_ev_rxq_flush_done,
@@ -464,6 +496,7 @@
 static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = {
 	.eec_initialized	= sfc_ev_initialized,
 	.eec_rx			= sfc_ev_nop_rx,
+	.eec_rx_ps		= sfc_ev_nop_rx_ps,
 	.eec_tx			= sfc_ev_tx,
 	.eec_exception		= sfc_ev_exception,
 	.eec_rxq_flush_done	= sfc_ev_nop_rxq_flush_done,
@@ -479,6 +512,7 @@
 static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = {
 	.eec_initialized	= sfc_ev_initialized,
 	.eec_rx			= sfc_ev_nop_rx,
+	.eec_rx_ps		= sfc_ev_nop_rx_ps,
 	.eec_tx			= sfc_ev_dp_tx,
 	.eec_exception		= sfc_ev_exception,
 	.eec_rxq_flush_done	= sfc_ev_nop_rxq_flush_done,
diff --git a/drivers/net/sfc/sfc_kvargs.h b/drivers/net/sfc/sfc_kvargs.h
index e4ee28f..c319bd2 100644
--- a/drivers/net/sfc/sfc_kvargs.h
+++ b/drivers/net/sfc/sfc_kvargs.h
@@ -55,11 +55,13 @@
 #define SFC_KVARG_DATAPATH_EFX		"efx"
 #define SFC_KVARG_DATAPATH_EF10		"ef10"
 #define SFC_KVARG_DATAPATH_EF10_SIMPLE	"ef10_simple"
+#define SFC_KVARG_DATAPATH_EF10_RX_PACKED	"ef10_packed"
 
 #define SFC_KVARG_RX_DATAPATH		"rx_datapath"
 #define SFC_KVARG_VALUES_RX_DATAPATH \
 	"[" SFC_KVARG_DATAPATH_EFX "|" \
-	    SFC_KVARG_DATAPATH_EF10 "]"
+	    SFC_KVARG_DATAPATH_EF10 "|" \
+	    SFC_KVARG_DATAPATH_EF10_RX_PACKED "]"
 
 #define SFC_KVARG_TX_DATAPATH		"tx_datapath"
 #define SFC_KVARG_VALUES_TX_DATAPATH \
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index 2dda5c7..ace02fa 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -861,6 +861,9 @@ struct sfc_dp_rx sfc_efx_rx = {
 	SFC_ASSERT(nb_rx_desc <= rxq_info->max_entries);
 	rxq_info->entries = nb_rx_desc;
 	rxq_info->type =
+		(sa->dp_rx->dp.hw_fw_caps &
+		 SFC_DP_HW_FW_CAP_RX_PACKED_STREAM_64K) ?
+		EFX_RXQ_TYPE_PACKED_STREAM_64K :
 		sa->eth_dev->data->dev_conf.rxmode.enable_scatter ?
 		EFX_RXQ_TYPE_SCATTER : EFX_RXQ_TYPE_DEFAULT;
 
@@ -881,7 +884,7 @@ struct sfc_dp_rx sfc_efx_rx = {
 	rxq_info->rxq = rxq;
 
 	rxq->evq = evq;
-	rxq->hw_index = sw_index;
+	rxq->hw_index = evq_index;
 	rxq->refill_threshold = rx_conf->rx_free_thresh;
 	rxq->refill_mb_pool = mb_pool;
 
diff --git a/drivers/net/sfc/sfc_rx.h b/drivers/net/sfc/sfc_rx.h
index 6407406..4c84150 100644
--- a/drivers/net/sfc/sfc_rx.h
+++ b/drivers/net/sfc/sfc_rx.h
@@ -87,7 +87,7 @@ struct sfc_rxq {
 static inline unsigned int
 sfc_rxq_sw_index_by_hw_index(unsigned int hw_index)
 {
-	return hw_index;
+	return hw_index - 1;
 }
 
 static inline unsigned int
-- 
1.8.2.3



More information about the dev mailing list