[dpdk-dev] [PATCH 17/38] bus/dpaa: integrate DPAA Bus with hardware blocks

[Shreyansh Jain]

Now that QBMAN (QMAN, BMAN) and FMAN drivers are available, this patch
integrates the DPAA Bus driver for using the drivers for scanning
devices and calling the PMD registered probe callbacks.

Signed-off-by: Hemant Agrawal <hemant.agrawal at nxp.com>
Signed-off-by: Shreyansh Jain <shreyansh.jain at nxp.com>
---
 drivers/bus/dpaa/dpaa_bus.c               | 236 ++++++++++++++++++++++++++++++
 drivers/bus/dpaa/rte_bus_dpaa_version.map |  39 +++++
 drivers/bus/dpaa/rte_dpaa_bus.h           |   6 +
 3 files changed, 281 insertions(+)

diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c
index 1c4627d..6a6baf1 100644
--- a/drivers/bus/dpaa/dpaa_bus.c
+++ b/drivers/bus/dpaa/dpaa_bus.c
@@ -64,9 +64,19 @@
 #include <rte_dpaa_bus.h>
 #include <rte_dpaa_logs.h>
 
+#include <fsl_usd.h>
+#include <fsl_qman.h>
+#include <fsl_bman.h>
+#include <of.h>
+#include <netcfg.h>
 
 struct rte_dpaa_bus rte_dpaa_bus;
+struct netcfg_info *dpaa_netcfg;
 
+/* define a variable to hold the portal_key, once created.*/
+pthread_key_t dpaa_portal_key;
+
+RTE_DEFINE_PER_LCORE(bool, _dpaa_io);
 
 static inline void
 dpaa_add_to_device_list(struct rte_dpaa_device *dev)
@@ -79,11 +89,237 @@ dpaa_remove_from_device_list(struct rte_dpaa_device *dev)
 {
 	TAILQ_INSERT_TAIL(&rte_dpaa_bus.device_list, dev, next);
 }
+
+static void dpaa_clean_device_list(void);
+
+static int
+dpaa_create_device_list(void)
+{
+	int i;
+	int ret;
+	struct rte_dpaa_device *dev;
+	struct fm_eth_port_cfg *cfg;
+	struct fman_if *fman_intf;
+
+	/* Creating Ethernet Devices */
+	for (i = 0; i < dpaa_netcfg->num_ethports; i++) {
+		dev = rte_zmalloc(NULL, sizeof(struct rte_dpaa_device),
+				  RTE_CACHE_LINE_SIZE);
+		if (!dev) {
+			DPAA_BUS_LOG(ERR, "Failed to allocate ETH devices");
+			ret = -ENOMEM;
+			goto cleanup;
+		}
+
+		cfg = &dpaa_netcfg->port_cfg[i];
+		fman_intf = cfg->fman_if;
+
+		/* Device identifiers */
+		dev->id.fman_id = fman_intf->fman_idx + 1;
+		dev->id.mac_id = fman_intf->mac_idx;
+		dev->id.device_type = FSL_DPAA_ETH;
+		dev->id.dev_id = i;
+
+		/* Create device name */
+		memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN);
+		sprintf(dev->name, "fm%d-mac%d", (fman_intf->fman_idx + 1),
+			fman_intf->mac_idx);
+		DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name);
+
+		dpaa_add_to_device_list(dev);
+	}
+
+	rte_dpaa_bus.device_count = i;
+
+	return 0;
+
+cleanup:
+	dpaa_clean_device_list();
+	return ret;
+}
+
+static void
+dpaa_clean_device_list(void)
+{
+	struct rte_dpaa_device *dev = NULL;
+	struct rte_dpaa_device *tdev = NULL;
+
+	TAILQ_FOREACH_SAFE(dev, &rte_dpaa_bus.device_list, next, tdev) {
+		TAILQ_REMOVE(&rte_dpaa_bus.device_list, dev, next);
+		rte_free(dev);
+		dev = NULL;
+	}
+}
+
+/** XXX move this function into a separate file */
+static int
+_dpaa_portal_init(void *arg)
+{
+	cpu_set_t cpuset;
+	pthread_t id;
+	uint32_t cpu = rte_lcore_id();
+	int ret;
+	struct dpaa_portal *dpaa_io_portal;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if ((uint64_t)arg == 1 || cpu == LCORE_ID_ANY)
+		cpu = rte_get_master_lcore();
+	/* if the core id is not supported */
+	else
+		if (cpu >= RTE_MAX_LCORE)
+			return -1;
+
+	/* Set CPU affinity for this thread */
+	CPU_ZERO(&cpuset);
+	CPU_SET(cpu, &cpuset);
+	id = pthread_self();
+	ret = pthread_setaffinity_np(id, sizeof(cpu_set_t), &cpuset);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "pthread_setaffinity_np failed on "
+			"core :%d with ret: %d", cpu, ret);
+		return ret;
+	}
+
+	/* Initialise bman thread portals */
+	ret = bman_thread_init();
+	if (ret) {
+		PMD_DRV_LOG(ERR, "bman_thread_init failed on "
+			"core %d with ret: %d", cpu, ret);
+		return ret;
+	}
+
+	PMD_DRV_LOG(DEBUG, "BMAN thread initialized");
+
+	/* Initialise qman thread portals */
+	ret = qman_thread_init();
+	if (ret) {
+		PMD_DRV_LOG(ERR, "bman_thread_init failed on "
+			"core %d with ret: %d", cpu, ret);
+		bman_thread_finish();
+		return ret;
+	}
+
+	PMD_DRV_LOG(DEBUG, "QMAN thread initialized");
+
+	dpaa_io_portal = rte_malloc(NULL, sizeof(struct dpaa_portal),
+				    RTE_CACHE_LINE_SIZE);
+	if (!dpaa_io_portal) {
+		PMD_DRV_LOG(ERR, "Unable to allocate memory");
+		bman_thread_finish();
+		qman_thread_finish();
+		return -ENOMEM;
+	}
+
+	dpaa_io_portal->qman_idx = qman_get_portal_index();
+	dpaa_io_portal->bman_idx = bman_get_portal_index();
+	dpaa_io_portal->tid = syscall(SYS_gettid);
+
+	ret = pthread_setspecific(dpaa_portal_key, (void *)dpaa_io_portal);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "pthread_setspecific failed on "
+			    "core %d with ret: %d", cpu, ret);
+		dpaa_portal_finish(NULL);
+
+		return ret;
+	}
+
+	RTE_PER_LCORE(_dpaa_io) = true;
+
+	PMD_DRV_LOG(DEBUG, "QMAN thread initialized");
+
+	return 0;
+}
+
+/*
+ * rte_dpaa_portal_init - Wrapper over _dpaa_portal_init with thread level check
+ * XXX Complete this
+ */
+int
+rte_dpaa_portal_init(void *arg)
+{
+	if (unlikely(!RTE_PER_LCORE(_dpaa_io)))
+		return _dpaa_portal_init(arg);
+
+	return 0;
+}
+
+void
+dpaa_portal_finish(void *arg)
+{
+	struct dpaa_portal *dpaa_io_portal = (struct dpaa_portal *)arg;
+
+	if (!dpaa_io_portal) {
+		PMD_DRV_LOG(DEBUG, "Portal already cleaned");
+		return;
+	}
+
+	bman_thread_finish();
+	qman_thread_finish();
+
+	pthread_setspecific(dpaa_portal_key, NULL);
+
+	rte_free(dpaa_io_portal);
+	dpaa_io_portal = NULL;
+
+	RTE_PER_LCORE(_dpaa_io) = false;
+}
+
 static int
 rte_dpaa_bus_scan(void)
 {
+	int ret;
+
 	PMD_INIT_FUNC_TRACE();
 
+	/* Load the device-tree driver */
+	ret = of_init();
+	if (ret) {
+		DPAA_BUS_LOG(ERR, "of_init failed with ret: %d", ret);
+		return -1;
+	}
+
+	/* Get the interface configurations from device-tree */
+	dpaa_netcfg = netcfg_acquire();
+	if (!dpaa_netcfg) {
+		DPAA_BUS_LOG(ERR, "netcfg_acquire failed");
+		return -EINVAL;
+	}
+
+	if (!dpaa_netcfg->num_ethports) {
+		DPAA_BUS_LOG(INFO, "no network interfaces available");
+		/* This is not an error */
+		return 0;
+	}
+
+	DPAA_BUS_LOG(DEBUG, "Bus: Address of netcfg=%p, Ethports=%d",
+				dpaa_netcfg, dpaa_netcfg->num_ethports);
+
+#ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
+	dump_netcfg(dpaa_netcfg);
+#endif
+
+	DPAA_BUS_LOG(DEBUG, "Number of devices = %d\n",
+		    dpaa_netcfg->num_ethports);
+	ret = dpaa_create_device_list();
+	if (ret) {
+		DPAA_BUS_LOG(ERR, "Unable to create device list. (%d)", ret);
+		return ret;
+	}
+
+	/* create the key, supplying a function that'll be invoked
+	 * when a portal affined thread will be deleted.
+	 */
+	ret = pthread_key_create(&dpaa_portal_key, dpaa_portal_finish);
+	if (ret) {
+		DPAA_BUS_LOG(DEBUG, "Unable to create pthread key. (%d)", ret);
+		dpaa_clean_device_list();
+		return ret;
+	}
+
+	DPAA_BUS_LOG(DEBUG, "dpaa_portal_key=%u, ret=%d\n",
+		    (unsigned int)dpaa_portal_key, ret);
+
 	return 0;
 }
 
diff --git a/drivers/bus/dpaa/rte_bus_dpaa_version.map b/drivers/bus/dpaa/rte_bus_dpaa_version.map
index 8c1ea65..3d4dc88 100644
--- a/drivers/bus/dpaa/rte_bus_dpaa_version.map
+++ b/drivers/bus/dpaa/rte_bus_dpaa_version.map
@@ -1,7 +1,46 @@
 DPDK_17.08 {
 	global:
 
+	bman_acquire;
+	bman_free_pool;
+	bman_get_params;
+	bman_new_pool;
+	bman_release;
+	dpaa_netcfg;
+	fman_ccsr_map_fd;
+	fman_dealloc_bufs_mask_hi;
+	fman_dealloc_bufs_mask_lo;
+	fman_if_disable_rx;
+	fman_if_enable_rx;
+	fman_if_discard_rx_errors;
+	fman_if_get_fc_threshold;
+	fman_if_get_fc_quanta;
+	fman_if_promiscuous_disable;
+	fman_if_promiscuous_enable;
+	fman_if_reset_mcast_filter_table;
+	fman_if_set_bp;
+	fman_if_set_fc_threshold;
+	fman_if_set_fc_quanta;
+	fman_if_set_fdoff;
+	fman_if_set_ic_params;
+	fman_if_set_maxfrm;
+	fman_if_set_mcast_filter_table;
+	fman_if_stats_get;
+	fman_if_stats_reset;
+	fm_mac_add_exact_match_mac_addr;
+	fm_mac_rem_exact_match_mac_addr;
+	netcfg_acquire;
+	netcfg_release;
+	qman_create_fq;
+	qman_dequeue;
+	qman_dqrr_consume;
+	qman_enqueue_multi;
+	qman_init_fq;
+	qman_set_vdq;
+	qman_reserve_fqid_range;
 	rte_dpaa_driver_register;
 	rte_dpaa_driver_unregister;
+	rte_dpaa_mem_ptov;
+	rte_dpaa_portal_init;
 
 };
diff --git a/drivers/bus/dpaa/rte_dpaa_bus.h b/drivers/bus/dpaa/rte_dpaa_bus.h
index 55e6793..34a7f4b 100644
--- a/drivers/bus/dpaa/rte_dpaa_bus.h
+++ b/drivers/bus/dpaa/rte_dpaa_bus.h
@@ -36,6 +36,12 @@
 #include <rte_bus.h>
 #include <rte_mempool.h>
 
+#include <fsl_usd.h>
+#include <fsl_qman.h>
+#include <fsl_bman.h>
+#include <of.h>
+#include <netcfg.h>
+
 #define FSL_DPAA_BUS_NAME	"FSL_DPAA_BUS"
 
 #define DEV_TO_DPAA_DEVICE(ptr)	\
-- 
2.7.4