[dpdk-dev] [PATCH 43/62] net/sfc: support the concept of RTE switch domains/ports

[Andrew Rybchenko]

From: Ivan Malov <ivan.malov at oktetlabs.ru>

A later patch will add support for RTE flow action PORT_ID
to MAE backend. The driver has to ensure that such actions
refer to RTE ethdev instances deployed on top of the same
physical device. Also, the driver needs a means to find
sibling RTE ethdev instances when parsing such actions.

In order to solve these problems, add a switch infrastructure
which allocates switch domains based on persistence of device
serial number string across switch ports included in a domain.
Explain mapping between RTE switch port IDs and MAE endpoints.

Signed-off-by: Ivan Malov <ivan.malov at oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko at solarflare.com>
Reviewed-by: Andy Moreton <amoreton at xilinx.com>
---
 drivers/net/sfc/meson.build  |   1 +
 drivers/net/sfc/sfc_ethdev.c |   7 +
 drivers/net/sfc/sfc_mae.c    |  35 +++++
 drivers/net/sfc/sfc_mae.h    |   6 +
 drivers/net/sfc/sfc_switch.c | 276 +++++++++++++++++++++++++++++++++++
 drivers/net/sfc/sfc_switch.h |  47 ++++++
 6 files changed, 372 insertions(+)
 create mode 100644 drivers/net/sfc/sfc_switch.c
 create mode 100644 drivers/net/sfc/sfc_switch.h

diff --git a/drivers/net/sfc/meson.build b/drivers/net/sfc/meson.build
index 7a893080cb..42b184c29e 100644
--- a/drivers/net/sfc/meson.build
+++ b/drivers/net/sfc/meson.build
@@ -47,6 +47,7 @@ sources = files(
 	'sfc_tx.c',
 	'sfc_tso.c',
 	'sfc_filter.c',
+	'sfc_switch.c',
 	'sfc_mae.c',
 	'sfc_flow.c',
 	'sfc_dp.c',
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index c0672083ec..107dd0f470 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -93,6 +93,7 @@ sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
 	struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 	struct sfc_rss *rss = &sas->rss;
+	struct sfc_mae *mae = &sa->mae;
 	uint64_t txq_offloads_def = 0;
 
 	sfc_log_init(sa, "entry");
@@ -187,6 +188,12 @@ sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP |
 			     RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP;
 
+	if (mae->status == SFC_MAE_STATUS_SUPPORTED) {
+		dev_info->switch_info.name = dev->device->driver->name;
+		dev_info->switch_info.domain_id = mae->switch_domain_id;
+		dev_info->switch_info.port_id = mae->switch_port_id;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/sfc/sfc_mae.c b/drivers/net/sfc/sfc_mae.c
index a5800ae722..64cd6b0e9b 100644
--- a/drivers/net/sfc/sfc_mae.c
+++ b/drivers/net/sfc/sfc_mae.c
@@ -15,11 +15,24 @@
 
 #include "sfc.h"
 #include "sfc_log.h"
+#include "sfc_switch.h"
+
+static int
+sfc_mae_assign_entity_mport(struct sfc_adapter *sa,
+			    efx_mport_sel_t *mportp)
+{
+	const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
+
+	return efx_mae_mport_by_pcie_function(encp->enc_pf, encp->enc_vf,
+					      mportp);
+}
 
 int
 sfc_mae_attach(struct sfc_adapter *sa)
 {
+	struct sfc_mae_switch_port_request switch_port_request = {0};
 	const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
+	efx_mport_sel_t entity_mport;
 	struct sfc_mae *mae = &sa->mae;
 	efx_mae_limits_t limits;
 	int rc;
@@ -41,6 +54,25 @@ sfc_mae_attach(struct sfc_adapter *sa)
 	if (rc != 0)
 		goto fail_mae_get_limits;
 
+	sfc_log_init(sa, "assign entity MPORT");
+	rc = sfc_mae_assign_entity_mport(sa, &entity_mport);
+	if (rc != 0)
+		goto fail_mae_assign_entity_mport;
+
+	sfc_log_init(sa, "assign RTE switch domain");
+	rc = sfc_mae_assign_switch_domain(sa, &mae->switch_domain_id);
+	if (rc != 0)
+		goto fail_mae_assign_switch_domain;
+
+	sfc_log_init(sa, "assign RTE switch port");
+	switch_port_request.type = SFC_MAE_SWITCH_PORT_INDEPENDENT;
+	switch_port_request.entity_mportp = &entity_mport;
+	rc = sfc_mae_assign_switch_port(mae->switch_domain_id,
+					&switch_port_request,
+					&mae->switch_port_id);
+	if (rc != 0)
+		goto fail_mae_assign_switch_port;
+
 	mae->status = SFC_MAE_STATUS_SUPPORTED;
 	mae->nb_action_rule_prios_max = limits.eml_max_n_action_prios;
 	TAILQ_INIT(&mae->action_sets);
@@ -49,6 +81,9 @@ sfc_mae_attach(struct sfc_adapter *sa)
 
 	return 0;
 
+fail_mae_assign_switch_port:
+fail_mae_assign_switch_domain:
+fail_mae_assign_entity_mport:
 fail_mae_get_limits:
 	efx_mae_fini(sa->nic);
 
diff --git a/drivers/net/sfc/sfc_mae.h b/drivers/net/sfc/sfc_mae.h
index 3c34d08f88..f92e62dcbe 100644
--- a/drivers/net/sfc/sfc_mae.h
+++ b/drivers/net/sfc/sfc_mae.h
@@ -12,6 +12,8 @@
 
 #include <stdbool.h>
 
+#include <rte_spinlock.h>
+
 #include "efx.h"
 
 #ifdef __cplusplus
@@ -45,6 +47,10 @@ enum sfc_mae_status {
 };
 
 struct sfc_mae {
+	/** Assigned switch domain identifier */
+	uint16_t			switch_domain_id;
+	/** Assigned switch port identifier */
+	uint16_t			switch_port_id;
 	/** NIC support for MAE status */
 	enum sfc_mae_status		status;
 	/** Priority level limit for MAE action rules */
diff --git a/drivers/net/sfc/sfc_switch.c b/drivers/net/sfc/sfc_switch.c
new file mode 100644
index 0000000000..395fc40263
--- /dev/null
+++ b/drivers/net/sfc/sfc_switch.c
@@ -0,0 +1,276 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2019-2020 Xilinx, Inc.
+ * Copyright(c) 2019 Solarflare Communications Inc.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ */
+
+#include <stdbool.h>
+
+#include <rte_common.h>
+#include <rte_spinlock.h>
+
+#include "efx.h"
+
+#include "sfc.h"
+#include "sfc_log.h"
+#include "sfc_switch.h"
+
+/**
+ * Switch port registry entry.
+ *
+ * Drivers aware of RTE switch domains also have to maintain RTE switch
+ * port IDs for RTE ethdev instances they operate. These IDs are supposed
+ * to stand for physical interconnect entities, in example, PCIe functions.
+ *
+ * In terms of MAE, a physical interconnect entity can be referred to using
+ * an MPORT selector, that is, a 32-bit value. RTE switch port IDs, in turn,
+ * are 16-bit values, so indirect mapping has to be maintained:
+ *
+ * +--------------------+          +---------------------------------------+
+ * | RTE switch port ID |  ------  |         MAE switch port entry         |
+ * +--------------------+          |         ---------------------         |
+ *                                 |                                       |
+ *                                 | Entity (PCIe function) MPORT selector |
+ *                                 |                   +                   |
+ *                                 |  Port type (independent/representor)  |
+ *                                 +---------------------------------------+
+ *
+ * This mapping comprises a port type to ensure that RTE switch port ID
+ * of a represented entity and that of its representor are different in
+ * the case when the entity gets plugged into DPDK and not into a guest.
+ */
+struct sfc_mae_switch_port {
+	TAILQ_ENTRY(sfc_mae_switch_port)	switch_domain_ports;
+
+	/** Entity (PCIe function) MPORT selector */
+	efx_mport_sel_t				entity_mport;
+	/** Port type (independent/representor) */
+	enum sfc_mae_switch_port_type		type;
+	/** RTE switch port ID */
+	uint16_t				id;
+};
+
+TAILQ_HEAD(sfc_mae_switch_ports, sfc_mae_switch_port);
+
+/**
+ * Switch domain registry entry.
+ *
+ * Even if an RTE ethdev instance gets unplugged, the corresponding
+ * entry in the switch port registry will not be removed because the
+ * entity (PCIe function) MPORT is static and cannot change. If this
+ * RTE ethdev gets plugged back, the entry will be reused, and
+ * RTE switch port ID will be the same.
+ */
+struct sfc_mae_switch_domain {
+	TAILQ_ENTRY(sfc_mae_switch_domain)	entries;
+
+	/** HW switch ID */
+	struct sfc_hw_switch_id			*hw_switch_id;
+	/** The number of ports in the switch port registry */
+	unsigned int				nb_ports;
+	/** Switch port registry */
+	struct sfc_mae_switch_ports		ports;
+	/** RTE switch domain ID allocated for a group of devices */
+	uint16_t				id;
+};
+
+TAILQ_HEAD(sfc_mae_switch_domains, sfc_mae_switch_domain);
+
+/**
+ * MAE representation of RTE switch infrastructure.
+ *
+ * It is possible that an RTE flow API client tries to insert a rule
+ * referencing an RTE ethdev deployed on top of a different physical
+ * device (it may belong to the same vendor or not). This particular
+ * driver/engine cannot support this and has to turn down such rules.
+ *
+ * Technically, it's HW switch identifier which, if queried for each
+ * RTE ethdev instance, indicates relationship between the instances.
+ * In the meantime, RTE flow API clients also need to somehow figure
+ * out relationship between RTE ethdev instances in advance.
+ *
+ * The concept of RTE switch domains resolves this issue. The driver
+ * maintains a static list of switch domains which is easy to browse,
+ * and each RTE ethdev fills RTE switch parameters in device
+ * information structure which is made available to clients.
+ *
+ * Even if all RTE ethdev instances belonging to a switch domain get
+ * unplugged, the corresponding entry in the switch domain registry
+ * will not be removed because the corresponding HW switch exists
+ * regardless of its ports being plugged to DPDK or kept aside.
+ * If a port gets plugged back to DPDK, the corresponding
+ * RTE ethdev will indicate the same RTE switch domain ID.
+ */
+struct sfc_mae_switch {
+	/** A lock to protect the whole structure */
+	rte_spinlock_t			lock;
+	/** Switch domain registry */
+	struct sfc_mae_switch_domains	domains;
+};
+
+static struct sfc_mae_switch sfc_mae_switch = {
+	.lock = RTE_SPINLOCK_INITIALIZER,
+	.domains = TAILQ_HEAD_INITIALIZER(sfc_mae_switch.domains),
+};
+
+
+/* This function expects to be called only when the lock is held */
+static struct sfc_mae_switch_domain *
+sfc_mae_find_switch_domain_by_id(uint16_t switch_domain_id)
+{
+	struct sfc_mae_switch_domain *domain;
+
+	SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock));
+
+	TAILQ_FOREACH(domain, &sfc_mae_switch.domains, entries) {
+		if (domain->id == switch_domain_id)
+			return domain;
+	}
+
+	return NULL;
+}
+
+/* This function expects to be called only when the lock is held */
+static struct sfc_mae_switch_domain *
+sfc_mae_find_switch_domain_by_hw_switch_id(const struct sfc_hw_switch_id *id)
+{
+	struct sfc_mae_switch_domain *domain;
+
+	SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock));
+
+	TAILQ_FOREACH(domain, &sfc_mae_switch.domains, entries) {
+		if (sfc_hw_switch_ids_equal(domain->hw_switch_id, id))
+			return domain;
+	}
+
+	return NULL;
+}
+
+int
+sfc_mae_assign_switch_domain(struct sfc_adapter *sa,
+			     uint16_t *switch_domain_id)
+{
+	struct sfc_hw_switch_id *hw_switch_id;
+	struct sfc_mae_switch_domain *domain;
+	int rc;
+
+	rte_spinlock_lock(&sfc_mae_switch.lock);
+
+	rc = sfc_hw_switch_id_init(sa, &hw_switch_id);
+	if (rc != 0)
+		goto fail_hw_switch_id_init;
+
+	domain = sfc_mae_find_switch_domain_by_hw_switch_id(hw_switch_id);
+	if (domain != NULL) {
+		sfc_hw_switch_id_fini(sa, hw_switch_id);
+		goto done;
+	}
+
+	domain = rte_zmalloc("sfc_mae_switch_domain", sizeof(*domain), 0);
+	if (domain == NULL) {
+		rc = ENOMEM;
+		goto fail_mem_alloc;
+	}
+
+	/*
+	 * This code belongs to driver init path, that is, negation is
+	 * done at the end of the path by sfc_eth_dev_init(). RTE APIs
+	 * negate error codes, so drop negation here.
+	 */
+	rc = -rte_eth_switch_domain_alloc(&domain->id);
+	if (rc != 0)
+		goto fail_domain_alloc;
+
+	domain->hw_switch_id = hw_switch_id;
+
+	TAILQ_INIT(&domain->ports);
+
+	TAILQ_INSERT_TAIL(&sfc_mae_switch.domains, domain, entries);
+
+done:
+	*switch_domain_id = domain->id;
+
+	rte_spinlock_unlock(&sfc_mae_switch.lock);
+
+	return 0;
+
+fail_domain_alloc:
+	rte_free(domain);
+
+fail_mem_alloc:
+	sfc_hw_switch_id_fini(sa, hw_switch_id);
+	rte_spinlock_unlock(&sfc_mae_switch.lock);
+
+fail_hw_switch_id_init:
+	return rc;
+}
+
+/* This function expects to be called only when the lock is held */
+static struct sfc_mae_switch_port *
+sfc_mae_find_switch_port_by_entity(const struct sfc_mae_switch_domain *domain,
+				   const efx_mport_sel_t *entity_mportp,
+				   enum sfc_mae_switch_port_type type)
+{
+	struct sfc_mae_switch_port *port;
+
+	SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock));
+
+	TAILQ_FOREACH(port, &domain->ports, switch_domain_ports) {
+		if (port->entity_mport.sel == entity_mportp->sel &&
+		    port->type == type)
+			return port;
+	}
+
+	return NULL;
+}
+
+int
+sfc_mae_assign_switch_port(uint16_t switch_domain_id,
+			   const struct sfc_mae_switch_port_request *req,
+			   uint16_t *switch_port_id)
+{
+	struct sfc_mae_switch_domain *domain;
+	struct sfc_mae_switch_port *port;
+	int rc;
+
+	rte_spinlock_lock(&sfc_mae_switch.lock);
+
+	domain = sfc_mae_find_switch_domain_by_id(switch_domain_id);
+	if (domain == NULL) {
+		rc = EINVAL;
+		goto fail_find_switch_domain_by_id;
+	}
+
+	port = sfc_mae_find_switch_port_by_entity(domain, req->entity_mportp,
+						  req->type);
+	if (port != NULL)
+		goto done;
+
+	port = rte_zmalloc("sfc_mae_switch_port", sizeof(*port), 0);
+	if (port == NULL) {
+		rc = ENOMEM;
+		goto fail_mem_alloc;
+	}
+
+	port->entity_mport.sel = req->entity_mportp->sel;
+	port->type = req->type;
+
+	port->id = (domain->nb_ports++);
+
+	TAILQ_INSERT_TAIL(&domain->ports, port, switch_domain_ports);
+
+done:
+	*switch_port_id = port->id;
+
+	rte_spinlock_unlock(&sfc_mae_switch.lock);
+
+	return 0;
+
+fail_mem_alloc:
+fail_find_switch_domain_by_id:
+	rte_spinlock_unlock(&sfc_mae_switch.lock);
+	return rc;
+}
diff --git a/drivers/net/sfc/sfc_switch.h b/drivers/net/sfc/sfc_switch.h
new file mode 100644
index 0000000000..9845ac8801
--- /dev/null
+++ b/drivers/net/sfc/sfc_switch.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2019-2020 Xilinx, Inc.
+ * Copyright(c) 2019 Solarflare Communications Inc.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ */
+
+#ifndef _SFC_SWITCH_H
+#define _SFC_SWITCH_H
+
+#include <stdint.h>
+
+#include "efx.h"
+
+#include "sfc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Options for MAE switch port type */
+enum sfc_mae_switch_port_type {
+	/**
+	 * The switch port is operated by a self-sufficient RTE ethdev
+	 * and thus refers to its underlying PCIe function
+	 */
+	SFC_MAE_SWITCH_PORT_INDEPENDENT = 0,
+};
+
+struct sfc_mae_switch_port_request {
+	enum sfc_mae_switch_port_type		type;
+	const efx_mport_sel_t			*entity_mportp;
+};
+
+int sfc_mae_assign_switch_domain(struct sfc_adapter *sa,
+				 uint16_t *switch_domain_id);
+
+int sfc_mae_assign_switch_port(uint16_t switch_domain_id,
+			       const struct sfc_mae_switch_port_request *req,
+			       uint16_t *switch_port_id);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_SWITCH_H */
-- 
2.17.1