[dpdk-dev] [PATCH dpdk-dev v3 1/2] eal: introduce rte-init queue for libraries initialization

[xiangxia.m.yue at gmail.com]

From: Tonghao Zhang <xiangxia.m.yue at gmail.com>

This patch introduces rte-init queue, users can register a callback
for theirs initialization. After the almost resource of DPDK are
available (e.g. memzone, ring), invoke eal_rte_init_run(). With this
way, users don't introduce additional codes in the eal layer.

Suggested-by: Olivier Matz <olivier.matz at 6wind.com>
Suggested-by: Jerin Jacob <jerinj at marvell.com>
Signed-off-by: Tonghao Zhang <xiangxia.m.yue at gmail.com>
---
This patch will be used for next patch.
and should be applied after http://patches.dpdk.org/patch/68119/
because of __rte_experimal
v3:
* add rte-init autotest
* add __rte_experimal prefix
* change return type of rte_init_register to int 
* update doc/api/doxy-api-index.md
* update rte_eal_version.map
* update comments
* remove unused *.h in rte_init.h
v2:
* rename rte_last_init_register ->rte_init_register
* rename rte_last_init struct ->rte_init
* rename rte_init_cb ->rte_init_cb_t 
* free the rte_init node when not used
* remove rte_init and others to eal_private.h
* add comments
* fix checkpatch warning
---
 app/test/Makefile                       |  1 +
 app/test/autotest_data.py               |  6 +++
 app/test/meson.build                    |  1 +
 app/test/test_rte_init.c                | 35 +++++++++++++
 doc/api/doxy-api-index.md               |  1 +
 lib/librte_eal/common/eal_common_init.c | 87 +++++++++++++++++++++++++++++++++
 lib/librte_eal/common/eal_private.h     | 23 +++++++++
 lib/librte_eal/common/meson.build       |  1 +
 lib/librte_eal/freebsd/Makefile         |  1 +
 lib/librte_eal/freebsd/eal.c            |  6 +++
 lib/librte_eal/include/rte_init.h       | 68 ++++++++++++++++++++++++++
 lib/librte_eal/linux/Makefile           |  1 +
 lib/librte_eal/linux/eal.c              |  6 +++
 lib/librte_eal/rte_eal_version.map      |  1 +
 14 files changed, 238 insertions(+)
 create mode 100644 app/test/test_rte_init.c
 create mode 100644 lib/librte_eal/common/eal_common_init.c
 create mode 100644 lib/librte_eal/include/rte_init.h

diff --git a/app/test/Makefile b/app/test/Makefile
index 1f080d162659..494606ad4226 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -148,6 +148,7 @@ SRCS-y += test_alarm.c
 SRCS-y += test_interrupts.c
 SRCS-y += test_version.c
 SRCS-y += test_func_reentrancy.c
+SRCS-y += test_rte_init.c
 
 SRCS-y += test_service_cores.c
 
diff --git a/app/test/autotest_data.py b/app/test/autotest_data.py
index 7b1d01389be4..d06c27d68b4a 100644
--- a/app/test/autotest_data.py
+++ b/app/test/autotest_data.py
@@ -267,6 +267,12 @@
         "Report":  None,
     },
     {
+        "Name":    "RTE INIT autotest",
+        "Command": "rte_init_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
         "Name":    "Atomics autotest",
         "Command": "atomic_autotest",
         "Func":    default_autotest,
diff --git a/app/test/meson.build b/app/test/meson.build
index 351d29cb657d..98a952a9285d 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -79,6 +79,7 @@ test_sources = files('commands.c',
 	'test_mempool.c',
 	'test_mempool_perf.c',
 	'test_memzone.c',
+	'test_rte_init.c',
 	'test_meter.c',
 	'test_metrics.c',
 	'test_mcslock.c',
diff --git a/app/test/test_rte_init.c b/app/test/test_rte_init.c
new file mode 100644
index 000000000000..f6143bc76310
--- /dev/null
+++ b/app/test/test_rte_init.c
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 DPDK Community
+ */
+
+#include <stdio.h>
+
+#include <rte_init.h>
+
+#include "test.h"
+
+static int
+rte_init_cb(__rte_unused const void *arg)
+{
+	return 0;
+}
+
+static int
+test_rte_init(void)
+{
+	printf("test rte-init register API\n");
+	if (rte_init_register(rte_init_cb, NULL, RTE_INIT_PRE) != 0)
+		return -1;
+
+	printf("test rte-init cb\n");
+	if (rte_init_register(NULL, NULL, RTE_INIT_PRE) != -EINVAL)
+		return -1;
+
+	printf("test rte-init type\n");
+	if (rte_init_register(NULL, NULL, 10) != -EINVAL)
+		return -1;
+
+	return 0;
+}
+
+REGISTER_TEST_COMMAND(rte_init_autotest, test_rte_init);
diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index dff496be0980..fa02c5f9f287 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -178,6 +178,7 @@ The public API headers are grouped by topics:
 
 - **misc**:
   [EAL config]         (@ref rte_eal.h),
+  [RTE INIT]           (@ref rte_init.h),
   [common]             (@ref rte_common.h),
   [experimental APIs]  (@ref rte_compat.h),
   [ABI versioning]     (@ref rte_function_versioning.h),
diff --git a/lib/librte_eal/common/eal_common_init.c b/lib/librte_eal/common/eal_common_init.c
new file mode 100644
index 000000000000..f5ae5c51b667
--- /dev/null
+++ b/lib/librte_eal/common/eal_common_init.c
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 DPDK Community
+ */
+
+#include <sys/queue.h>
+
+#include <rte_init.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+
+#include "eal_private.h"
+
+static struct rte_init_list rte_init_list =
+	TAILQ_HEAD_INITIALIZER(rte_init_list);
+
+int
+rte_init_register(rte_init_cb_t cb, const void *arg, enum rte_init_type type)
+{
+	struct rte_init *last;
+
+	if (cb == NULL) {
+		RTE_LOG(ERR, EAL, "RTE INIT cb is NULL.\n");
+		return -EINVAL;
+	}
+
+	if (type != RTE_INIT_PRE && type != RTE_INIT_POST) {
+		RTE_LOG(ERR, EAL, "RTE INIT type is invalid.\n");
+		return -EINVAL;
+	}
+
+	last = malloc(sizeof(*last));
+	if (last == NULL) {
+		RTE_LOG(ERR, EAL,
+			"Allocate memory for rte_init node failed.\n");
+		return -ENOMEM;
+	}
+
+	last->type = type;
+	last->arg = arg;
+	last->cb = cb;
+
+	TAILQ_INSERT_TAIL(&rte_init_list, last, next);
+
+	return 0;
+}
+
+static int
+eal_rte_init_run_type(enum rte_init_type type)
+{
+	struct rte_init *last;
+	int ret;
+
+	TAILQ_FOREACH(last, &rte_init_list, next) {
+		if (last->type != type)
+			continue;
+
+		ret = last->cb(last->arg);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+int
+eal_rte_init_run(void)
+{
+	struct rte_init *last;
+	struct rte_init *tmp;
+	int ret;
+
+	ret = eal_rte_init_run_type(RTE_INIT_PRE);
+	if (ret < 0)
+		return ret;
+
+	ret = eal_rte_init_run_type(RTE_INIT_POST);
+	if (ret < 0)
+		return ret;
+
+	/* Free rte_init node, not used anymore. */
+	TAILQ_FOREACH_SAFE(last, &rte_init_list, next, tmp) {
+		TAILQ_REMOVE(&rte_init_list, last, next);
+		free(last);
+	}
+
+	return 0;
+}
diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h
index ddcfbe2e4442..f0bcc977eb9d 100644
--- a/lib/librte_eal/common/eal_private.h
+++ b/lib/librte_eal/common/eal_private.h
@@ -11,6 +11,7 @@
 
 #include <rte_dev.h>
 #include <rte_lcore.h>
+#include <rte_init.h>
 
 /**
  * Structure storing internal configuration (per-lcore)
@@ -60,6 +61,28 @@ struct rte_config {
 } __attribute__((__packed__));
 
 /**
+ * A structure describing a generic initialization.
+ */
+struct rte_init {
+	TAILQ_ENTRY(rte_init) next;
+	enum rte_init_type type;
+	rte_init_cb_t cb;
+	const void *arg;
+};
+
+/** Double linked list of rte_init. */
+TAILQ_HEAD(rte_init_list, rte_init);
+
+/**
+ * Run the callback registered in the global double linked list.
+ *
+ * @return
+ *   - 0 on success
+ *   - Negative on error
+ */
+int eal_rte_init_run(void);
+
+/**
  * Get the global configuration structure.
  *
  * @return
diff --git a/lib/librte_eal/common/meson.build b/lib/librte_eal/common/meson.build
index 02d9280cc351..ad0ce6a19015 100644
--- a/lib/librte_eal/common/meson.build
+++ b/lib/librte_eal/common/meson.build
@@ -43,6 +43,7 @@ sources += files(
 	'eal_common_thread.c',
 	'eal_common_timer.c',
 	'eal_common_uuid.c',
+	'eal_common_init.c',
 	'hotplug_mp.c',
 	'malloc_elem.c',
 	'malloc_heap.c',
diff --git a/lib/librte_eal/freebsd/Makefile b/lib/librte_eal/freebsd/Makefile
index e5d4d8ff260b..89c5649f16e6 100644
--- a/lib/librte_eal/freebsd/Makefile
+++ b/lib/librte_eal/freebsd/Makefile
@@ -60,6 +60,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_common_thread.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_common_proc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_common_fbarray.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_common_uuid.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_common_init.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += hotplug_mp.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += malloc_elem.c
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e69be..63af98cb6ca7 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -874,6 +874,12 @@ static void rte_eal_init_alert(const char *msg)
 
 	eal_check_mem_on_local_socket();
 
+	if (eal_rte_init_run() < 0) {
+		rte_eal_init_alert("Cannot init objects in rte-init queue");
+		rte_errno = EFAULT;
+		return -1;
+	}
+
 	eal_thread_init_master(rte_config.master_lcore);
 
 	ret = eal_thread_dump_affinity(cpuset, sizeof(cpuset));
diff --git a/lib/librte_eal/include/rte_init.h b/lib/librte_eal/include/rte_init.h
new file mode 100644
index 000000000000..83c7fd47daec
--- /dev/null
+++ b/lib/librte_eal/include/rte_init.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 DPDK Community
+ */
+
+#ifndef _RTE_INIT_H_
+#define _RTE_INIT_H_
+
+#include <rte_compat.h>
+
+/**
+ * @file
+ *
+ * RTE INIT Registration Interface
+ *
+ * This file exposes API for other libraries initialization callback.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Implementation specific callback function which is
+ * responsible for specificed initialization.
+ *
+ * This is invoked when almost resources are available.
+ *
+ * @return
+ *	0 for successfully invoked
+ *	Negative for unsuccessfully invoked with error value
+ */
+typedef int (*rte_init_cb_t)(const void *arg);
+
+/**
+ * The RTE INIT type of callback function registered.
+ */
+enum rte_init_type {
+	RTE_INIT_PRE, /**< RTE INITs are invoked with high priority. */
+	RTE_INIT_POST /**< Last RTE INITs invoked. */
+};
+
+/**
+ * Register a rte_init callback.
+ *
+ * @param cb
+ *   A pointer to a rte_init_cb structure, which will be invoked
+ *   in rte_eal_init().
+ *
+ * @param arg
+ *   The cb will use that as param.
+ *
+ * @param type
+ *   The type of rte_init registered.
+ *
+ * @return
+ *	0 for success register callback.
+ *	-EINVAL one of the parameters was invalid.
+ *	-ENOMEM no appropriate memory allocated.
+ */
+__rte_experimental
+int rte_init_register(rte_init_cb_t cb, const void *arg,
+		       enum rte_init_type type);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_INIT_H_ */
diff --git a/lib/librte_eal/linux/Makefile b/lib/librte_eal/linux/Makefile
index e5f44959c66c..918d94bec91e 100644
--- a/lib/librte_eal/linux/Makefile
+++ b/lib/librte_eal/linux/Makefile
@@ -67,6 +67,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_thread.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_proc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_fbarray.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_uuid.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_init.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += hotplug_mp.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += malloc_elem.c
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index 9530ee55f8e3..dd0c2589f6c4 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -1203,6 +1203,12 @@ static void rte_eal_init_alert(const char *msg)
 
 	eal_check_mem_on_local_socket();
 
+	if (eal_rte_init_run() < 0) {
+		rte_eal_init_alert("Cannot init objects in rte-init queue");
+		rte_errno = EFAULT;
+		return -1;
+	}
+
 	eal_thread_init_master(rte_config.master_lcore);
 
 	ret = eal_thread_dump_affinity(cpuset, sizeof(cpuset));
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index f9ede5b41c69..23aa30f542db 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -338,4 +338,5 @@ EXPERIMENTAL {
 
 	# added in 20.05
 	rte_log_can_log;
+	rte_init_register;
 };
-- 
1.8.3.1