[dpdk-dev] [PATCH 1/4] net/cxgbe: add API to program hardware layer 2 table

Rahul Lakkireddy rahul.lakkireddy at chelsio.com
Mon Aug 27 14:52:29 CEST 2018


From: Shagun Agrawal <shaguna at chelsio.com>

Add API to program and manage hardware Layer 2 Table. L2T holds
information necessary to rewrite specific fields in packet, such
as destination MAC address and vlan id.

Signed-off-by: Shagun Agrawal <shaguna at chelsio.com>
Signed-off-by: Rahul Lakkireddy <rahul.lakkireddy at chelsio.com>
---
 drivers/net/cxgbe/Makefile              |   1 +
 drivers/net/cxgbe/base/adapter.h        |   3 +
 drivers/net/cxgbe/base/t4_msg.h         |  34 +++++
 drivers/net/cxgbe/base/t4fw_interface.h |   2 +
 drivers/net/cxgbe/cxgbe_filter.h        |   1 +
 drivers/net/cxgbe/cxgbe_main.c          |  30 ++++-
 drivers/net/cxgbe/l2t.c                 | 227 ++++++++++++++++++++++++++++++++
 drivers/net/cxgbe/l2t.h                 |  57 ++++++++
 drivers/net/cxgbe/meson.build           |   1 +
 9 files changed, 349 insertions(+), 7 deletions(-)
 create mode 100644 drivers/net/cxgbe/l2t.c
 create mode 100644 drivers/net/cxgbe/l2t.h

diff --git a/drivers/net/cxgbe/Makefile b/drivers/net/cxgbe/Makefile
index 5d66c4b3a..d75b070f3 100644
--- a/drivers/net/cxgbe/Makefile
+++ b/drivers/net/cxgbe/Makefile
@@ -53,6 +53,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_filter.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4_hw.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += clip_tbl.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += l2t.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4vf_hw.c
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/cxgbe/base/adapter.h b/drivers/net/cxgbe/base/adapter.h
index e98dd2182..9f4a9653c 100644
--- a/drivers/net/cxgbe/base/adapter.h
+++ b/drivers/net/cxgbe/base/adapter.h
@@ -324,7 +324,10 @@ struct adapter {
 
 	unsigned int clipt_start; /* CLIP table start */
 	unsigned int clipt_end;   /* CLIP table end */
+	unsigned int l2t_start;   /* Layer 2 table start */
+	unsigned int l2t_end;     /* Layer 2 table end */
 	struct clip_tbl *clipt;   /* CLIP table */
+	struct l2t_data *l2t;     /* Layer 2 table */
 
 	struct tid_info tids;     /* Info used to access TID related tables */
 };
diff --git a/drivers/net/cxgbe/base/t4_msg.h b/drivers/net/cxgbe/base/t4_msg.h
index 5d433c91c..094a153f2 100644
--- a/drivers/net/cxgbe/base/t4_msg.h
+++ b/drivers/net/cxgbe/base/t4_msg.h
@@ -11,7 +11,9 @@ enum {
 	CPL_SET_TCB_FIELD     = 0x5,
 	CPL_ABORT_REQ         = 0xA,
 	CPL_ABORT_RPL         = 0xB,
+	CPL_L2T_WRITE_REQ     = 0x12,
 	CPL_TID_RELEASE       = 0x1A,
+	CPL_L2T_WRITE_RPL     = 0x23,
 	CPL_ACT_OPEN_RPL      = 0x25,
 	CPL_ABORT_RPL_RSS     = 0x2D,
 	CPL_SET_TCB_RPL       = 0x3A,
@@ -66,6 +68,9 @@ union opcode_tid {
 #define M_TID_TID    0x3fff
 #define G_TID_TID(x) (((x) >> S_TID_TID) & M_TID_TID)
 
+#define S_TID_QID    14
+#define V_TID_QID(x) ((x) << S_TID_QID)
+
 struct rss_header {
 	__u8 opcode;
 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
@@ -421,6 +426,35 @@ struct cpl_rx_pkt {
 	__be16 err_vec;
 };
 
+struct cpl_l2t_write_req {
+	WR_HDR;
+	union opcode_tid ot;
+	__be16 params;
+	__be16 l2t_idx;
+	__be16 vlan;
+	__u8   dst_mac[6];
+};
+
+/* cpl_l2t_write_req.params fields */
+#define S_L2T_W_PORT    8
+#define V_L2T_W_PORT(x) ((x) << S_L2T_W_PORT)
+
+#define S_L2T_W_LPBK    10
+#define V_L2T_W_LPBK(x) ((x) << S_L2T_W_LPBK)
+
+#define S_L2T_W_ARPMISS         11
+#define V_L2T_W_ARPMISS(x)      ((x) << S_L2T_W_ARPMISS)
+
+#define S_L2T_W_NOREPLY    15
+#define V_L2T_W_NOREPLY(x) ((x) << S_L2T_W_NOREPLY)
+
+struct cpl_l2t_write_rpl {
+	RSS_HDR
+	union opcode_tid ot;
+	__u8 status;
+	__u8 rsvd[3];
+};
+
 /* rx_pkt.l2info fields */
 #define S_RXF_UDP    22
 #define V_RXF_UDP(x) ((x) << S_RXF_UDP)
diff --git a/drivers/net/cxgbe/base/t4fw_interface.h b/drivers/net/cxgbe/base/t4fw_interface.h
index e80b58a32..1c08637bb 100644
--- a/drivers/net/cxgbe/base/t4fw_interface.h
+++ b/drivers/net/cxgbe/base/t4fw_interface.h
@@ -665,6 +665,8 @@ enum fw_params_param_pfvf {
 	FW_PARAMS_PARAM_PFVF_CLIP_END = 0x04,
 	FW_PARAMS_PARAM_PFVF_FILTER_START = 0x05,
 	FW_PARAMS_PARAM_PFVF_FILTER_END = 0x06,
+	FW_PARAMS_PARAM_PFVF_L2T_START = 0x13,
+	FW_PARAMS_PARAM_PFVF_L2T_END = 0x14,
 	FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP = 0x31,
 	FW_PARAMS_PARAM_PFVF_PORT_CAPS32 = 0x3A
 };
diff --git a/drivers/net/cxgbe/cxgbe_filter.h b/drivers/net/cxgbe/cxgbe_filter.h
index af8fa7529..be12e231a 100644
--- a/drivers/net/cxgbe/cxgbe_filter.h
+++ b/drivers/net/cxgbe/cxgbe_filter.h
@@ -145,6 +145,7 @@ struct filter_entry {
 	u32 pending:1;              /* filter action is pending FW reply */
 	struct filter_ctx *ctx;     /* caller's completion hook */
 	struct clip_entry *clipt;   /* CLIP Table entry for IPv6 */
+	struct l2t_entry *l2t;      /* Layer Two Table entry for dmac */
 	struct rte_eth_dev *dev;    /* Port's rte eth device */
 	void *private;              /* For use by apps using filter_entry */
 
diff --git a/drivers/net/cxgbe/cxgbe_main.c b/drivers/net/cxgbe/cxgbe_main.c
index c3938e8db..be2bc4213 100644
--- a/drivers/net/cxgbe/cxgbe_main.c
+++ b/drivers/net/cxgbe/cxgbe_main.c
@@ -38,6 +38,7 @@
 #include "t4_msg.h"
 #include "cxgbe.h"
 #include "clip_tbl.h"
+#include "l2t.h"
 
 /**
  * Allocate a chunk of memory. The allocated memory is cleared.
@@ -99,6 +100,10 @@ static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 		const struct cpl_act_open_rpl *p = (const void *)rsp;
 
 		hash_filter_rpl(q->adapter, p);
+	} else if (opcode == CPL_L2T_WRITE_RPL) {
+		const struct cpl_l2t_write_rpl *p = (const void *)rsp;
+
+		do_l2t_write_rpl(q->adapter, p);
 	} else {
 		dev_err(adapter, "unexpected CPL %#x on FW event queue\n",
 			opcode);
@@ -1135,13 +1140,17 @@ static int adap_init0(struct adapter *adap)
 	 V_FW_PARAMS_PARAM_Y(0) | \
 	 V_FW_PARAMS_PARAM_Z(0))
 
-	params[0] = FW_PARAM_PFVF(FILTER_START);
-	params[1] = FW_PARAM_PFVF(FILTER_END);
-	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
+	params[0] = FW_PARAM_PFVF(L2T_START);
+	params[1] = FW_PARAM_PFVF(L2T_END);
+	params[2] = FW_PARAM_PFVF(FILTER_START);
+	params[3] = FW_PARAM_PFVF(FILTER_END);
+	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 4, params, val);
 	if (ret < 0)
 		goto bye;
-	adap->tids.ftid_base = val[0];
-	adap->tids.nftids = val[1] - val[0] + 1;
+	adap->l2t_start = val[0];
+	adap->l2t_end = val[1];
+	adap->tids.ftid_base = val[2];
+	adap->tids.nftids = val[3] - val[2] + 1;
 
 	params[0] = FW_PARAM_PFVF(CLIP_START);
 	params[1] = FW_PARAM_PFVF(CLIP_END);
@@ -1679,10 +1688,11 @@ void cxgbe_close(struct adapter *adapter)
 	int i;
 
 	if (adapter->flags & FULL_INIT_DONE) {
-		if (is_pf4(adapter))
-			t4_intr_disable(adapter);
 		tid_free(&adapter->tids);
 		t4_cleanup_clip_tbl(adapter);
+		t4_cleanup_l2t(adapter);
+		if (is_pf4(adapter))
+			t4_intr_disable(adapter);
 		t4_sge_tx_monitor_stop(adapter);
 		t4_free_sge_resources(adapter);
 		for_each_port(adapter, i) {
@@ -1855,6 +1865,12 @@ int cxgbe_probe(struct adapter *adapter)
 		dev_warn(adapter, "could not allocate CLIP. Continuing\n");
 	}
 
+	adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end);
+	if (!adapter->l2t) {
+		/* We tolerate a lack of L2T, giving up some functionality */
+		dev_warn(adapter, "could not allocate L2T. Continuing\n");
+	}
+
 	if (tid_init(&adapter->tids) < 0) {
 		/* Disable filtering support */
 		dev_warn(adapter, "could not allocate TID table, "
diff --git a/drivers/net/cxgbe/l2t.c b/drivers/net/cxgbe/l2t.c
new file mode 100644
index 000000000..814188fea
--- /dev/null
+++ b/drivers/net/cxgbe/l2t.c
@@ -0,0 +1,227 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#include "common.h"
+#include "l2t.h"
+
+/**
+ * cxgbe_l2t_release - Release associated L2T entry
+ * @e: L2T entry to release
+ *
+ * Releases ref count and frees up an L2T entry from L2T table
+ */
+void cxgbe_l2t_release(struct l2t_entry *e)
+{
+	if (rte_atomic32_read(&e->refcnt) != 0)
+		rte_atomic32_dec(&e->refcnt);
+}
+
+/**
+ * Process a CPL_L2T_WRITE_RPL. Note that the TID in the reply is really
+ * the L2T index it refers to.
+ */
+void do_l2t_write_rpl(struct adapter *adap, const struct cpl_l2t_write_rpl *rpl)
+{
+	struct l2t_data *d = adap->l2t;
+	unsigned int tid = GET_TID(rpl);
+	unsigned int l2t_idx = tid % L2T_SIZE;
+
+	if (unlikely(rpl->status != CPL_ERR_NONE)) {
+		dev_err(adap,
+			"Unexpected L2T_WRITE_RPL status %u for entry %u\n",
+			rpl->status, l2t_idx);
+		return;
+	}
+
+	if (tid & F_SYNC_WR) {
+		struct l2t_entry *e = &d->l2tab[l2t_idx - d->l2t_start];
+
+		t4_os_lock(&e->lock);
+		if (e->state != L2T_STATE_SWITCHING)
+			e->state = L2T_STATE_VALID;
+		t4_os_unlock(&e->lock);
+	}
+}
+
+/**
+ * Write an L2T entry.  Must be called with the entry locked.
+ * The write may be synchronous or asynchronous.
+ */
+static int write_l2e(struct rte_eth_dev *dev, struct l2t_entry *e, int sync,
+		     bool loopback, bool arpmiss)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct l2t_data *d = adap->l2t;
+	struct rte_mbuf *mbuf;
+	struct cpl_l2t_write_req *req;
+	struct sge_ctrl_txq *ctrlq;
+	unsigned int l2t_idx = e->idx + d->l2t_start;
+	unsigned int port_id = ethdev2pinfo(dev)->port_id;
+
+	ctrlq = &adap->sge.ctrlq[port_id];
+	mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+	if (!mbuf)
+		return -ENOMEM;
+
+	mbuf->data_len = sizeof(*req);
+	mbuf->pkt_len = mbuf->data_len;
+
+	req = rte_pktmbuf_mtod(mbuf, struct cpl_l2t_write_req *);
+	INIT_TP_WR(req, 0);
+
+	OPCODE_TID(req) =
+		cpu_to_be32(MK_OPCODE_TID(CPL_L2T_WRITE_REQ,
+					  l2t_idx | V_SYNC_WR(sync) |
+					  V_TID_QID(adap->sge.fw_evtq.abs_id)));
+	req->params = cpu_to_be16(V_L2T_W_PORT(e->lport) |
+				  V_L2T_W_LPBK(loopback) |
+				  V_L2T_W_ARPMISS(arpmiss) |
+				  V_L2T_W_NOREPLY(!sync));
+	req->l2t_idx = cpu_to_be16(l2t_idx);
+	req->vlan = cpu_to_be16(e->vlan);
+	rte_memcpy(req->dst_mac, e->dmac, ETHER_ADDR_LEN);
+
+	if (loopback)
+		memset(req->dst_mac, 0, ETHER_ADDR_LEN);
+
+	t4_mgmt_tx(ctrlq, mbuf);
+
+	if (sync && e->state != L2T_STATE_SWITCHING)
+		e->state = L2T_STATE_SYNC_WRITE;
+
+	return 0;
+}
+
+/**
+ * find_or_alloc_l2e - Find/Allocate a free L2T entry
+ * @d: L2T table
+ * @vlan: VLAN id to compare/add
+ * @port: port id to compare/add
+ * @dmac: Destination MAC address to compare/add
+ * Returns pointer to the L2T entry found/created
+ *
+ * Finds/Allocates an L2T entry to be used by switching rule of a filter.
+ */
+static struct l2t_entry *find_or_alloc_l2e(struct l2t_data *d, u16 vlan,
+					   u8 port, u8 *dmac)
+{
+	struct l2t_entry *end, *e;
+	struct l2t_entry *first_free = NULL;
+
+	for (e = &d->l2tab[0], end = &d->l2tab[d->l2t_size]; e != end; ++e) {
+		if (rte_atomic32_read(&e->refcnt) == 0) {
+			if (!first_free)
+				first_free = e;
+		} else {
+			if (e->state == L2T_STATE_SWITCHING) {
+				if ((!memcmp(e->dmac, dmac, ETHER_ADDR_LEN)) &&
+				    e->vlan == vlan && e->lport == port)
+					goto exists;
+			}
+		}
+	}
+
+	if (first_free) {
+		e = first_free;
+		goto found;
+	}
+
+	return NULL;
+
+found:
+	e->state = L2T_STATE_UNUSED;
+
+exists:
+	return e;
+}
+
+static struct l2t_entry *t4_l2t_alloc_switching(struct rte_eth_dev *dev,
+						u16 vlan, u8 port,
+						u8 *eth_addr)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct l2t_data *d = adap->l2t;
+	struct l2t_entry *e;
+	int ret = 0;
+
+	t4_os_write_lock(&d->lock);
+	e = find_or_alloc_l2e(d, vlan, port, eth_addr);
+	if (e) {
+		t4_os_lock(&e->lock);
+		if (!rte_atomic32_read(&e->refcnt)) {
+			e->state = L2T_STATE_SWITCHING;
+			e->vlan = vlan;
+			e->lport = port;
+			rte_memcpy(e->dmac, eth_addr, ETHER_ADDR_LEN);
+			rte_atomic32_set(&e->refcnt, 1);
+			ret = write_l2e(dev, e, 0, !L2T_LPBK, !L2T_ARPMISS);
+			if (ret < 0)
+				dev_debug(adap, "Failed to write L2T entry: %d",
+					  ret);
+		} else {
+			rte_atomic32_inc(&e->refcnt);
+		}
+		t4_os_unlock(&e->lock);
+	}
+	t4_os_write_unlock(&d->lock);
+
+	return ret ? NULL : e;
+}
+
+/**
+ * cxgbe_l2t_alloc_switching - Allocate a L2T entry for switching rule
+ * @dev: rte_eth_dev pointer
+ * @vlan: VLAN Id
+ * @port: Associated port
+ * @dmac: Destination MAC address to add to L2T
+ * Returns pointer to the allocated l2t entry
+ *
+ * Allocates a L2T entry for use by switching rule of a filter
+ */
+struct l2t_entry *cxgbe_l2t_alloc_switching(struct rte_eth_dev *dev, u16 vlan,
+					    u8 port, u8 *dmac)
+{
+	return t4_l2t_alloc_switching(dev, vlan, port, dmac);
+}
+
+/**
+ * Initialize L2 Table
+ */
+struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end)
+{
+	unsigned int l2t_size;
+	unsigned int i;
+	struct l2t_data *d;
+
+	if (l2t_start >= l2t_end || l2t_end >= L2T_SIZE)
+		return NULL;
+	l2t_size = l2t_end - l2t_start + 1;
+
+	d = t4_os_alloc(sizeof(*d) + l2t_size * sizeof(struct l2t_entry));
+	if (!d)
+		return NULL;
+
+	d->l2t_start = l2t_start;
+	d->l2t_size = l2t_size;
+
+	t4_os_rwlock_init(&d->lock);
+
+	for (i = 0; i < d->l2t_size; ++i) {
+		d->l2tab[i].idx = i;
+		d->l2tab[i].state = L2T_STATE_UNUSED;
+		t4_os_lock_init(&d->l2tab[i].lock);
+		rte_atomic32_set(&d->l2tab[i].refcnt, 0);
+	}
+
+	return d;
+}
+
+/**
+ * Cleanup L2 Table
+ */
+void t4_cleanup_l2t(struct adapter *adap)
+{
+	if (adap->l2t)
+		t4_os_free(adap->l2t);
+}
diff --git a/drivers/net/cxgbe/l2t.h b/drivers/net/cxgbe/l2t.h
new file mode 100644
index 000000000..22a34e388
--- /dev/null
+++ b/drivers/net/cxgbe/l2t.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#ifndef _CXGBE_L2T_H_
+#define _CXGBE_L2T_H_
+
+#include "t4_msg.h"
+
+enum {
+	L2T_SIZE = 4096       /* # of L2T entries */
+};
+
+enum {
+	L2T_STATE_VALID,      /* entry is up to date */
+	L2T_STATE_SYNC_WRITE, /* synchronous write of entry underway */
+
+	/* when state is one of the below the entry is not hashed */
+	L2T_STATE_SWITCHING,  /* entry is being used by a switching filter */
+	L2T_STATE_UNUSED      /* entry not in use */
+};
+
+/*
+ * State for the corresponding entry of the HW L2 table.
+ */
+struct l2t_entry {
+	u16 state;                  /* entry state */
+	u16 idx;                    /* entry index within in-memory table */
+	u16 vlan;                   /* VLAN TCI (id: bits 0-11, prio: 13-15 */
+	u8  lport;                  /* destination port */
+	u8  dmac[ETHER_ADDR_LEN];   /* destination MAC address */
+	rte_spinlock_t lock;        /* entry lock */
+	rte_atomic32_t refcnt;      /* entry reference count */
+};
+
+struct l2t_data {
+	unsigned int l2t_start;     /* start index of our piece of the L2T */
+	unsigned int l2t_size;      /* number of entries in l2tab */
+	rte_rwlock_t lock;          /* table rw lock */
+	struct l2t_entry l2tab[0];  /* MUST BE LAST */
+};
+
+#define L2T_LPBK	true
+#define L2T_ARPMISS	true
+
+/* identifies sync vs async L2T_WRITE_REQs */
+#define S_SYNC_WR    12
+#define V_SYNC_WR(x) ((x) << S_SYNC_WR)
+#define F_SYNC_WR    V_SYNC_WR(1)
+
+struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end);
+void t4_cleanup_l2t(struct adapter *adap);
+struct l2t_entry *cxgbe_l2t_alloc_switching(struct rte_eth_dev *dev, u16 vlan,
+					    u8 port, u8 *dmac);
+void cxgbe_l2t_release(struct l2t_entry *e);
+void do_l2t_write_rpl(struct adapter *p, const struct cpl_l2t_write_rpl *rpl);
+#endif /* _CXGBE_L2T_H_ */
diff --git a/drivers/net/cxgbe/meson.build b/drivers/net/cxgbe/meson.build
index 7c69a34b1..f6a442cb8 100644
--- a/drivers/net/cxgbe/meson.build
+++ b/drivers/net/cxgbe/meson.build
@@ -9,6 +9,7 @@ sources = files('cxgbe_ethdev.c',
 	'cxgbe_filter.c',
 	'cxgbe_flow.c',
 	'clip_tbl.c',
+	'l2t.c',
 	'base/t4_hw.c',
 	'base/t4vf_hw.c')
 includes += include_directories('base')
-- 
2.14.1



More information about the dev mailing list