[dpdk-dev] [PATCH v4 06/18] telemetry: introduce new telemetry functionality

[Wiles, Keith]

> On Apr 24, 2020, at 7:41 AM, Power, Ciara <ciara.power at intel.com> wrote:
> 
> From: Bruce Richardson <bruce.richardson at intel.com>
> 
> This patch introduces a new telemetry connection socket and handling
> functionality. Like the existing telemetry implementation (which is
> unaffected by this change) it uses a unix socket, but unlike the
> existing one it does not have a fixed list of commands - instead
> libraries or applications can register telemetry commands and callbacks
> to provide a full-extensible solution for all kinds of telemetry across
> DPDK.
> 
> Signed-off-by: Bruce Richardson <bruce.richardson at intel.com>
> Signed-off-by: Ciara Power <ciara.power at intel.com>
> 
> ---
> v4:
>  - Added initial non-json internal representation prototypes.
> v2:
>  - Added DPDK information message on client connection.
>  - Added /info command to get DPDK information post-connect.
>  - Replaced pid in socket name with v2.
>  - Increased output buffer size to 16k.
>  - Telemetry default callbacks now registered by init function.
> ---
> lib/librte_telemetry/Makefile                 |   4 +
> lib/librte_telemetry/meson.build              |   5 +-
> lib/librte_telemetry/rte_telemetry.c          |   3 +
> lib/librte_telemetry/rte_telemetry.h          |  95 ++++-
> .../rte_telemetry_version.map                 |   2 +
> lib/librte_telemetry/telemetry.c              | 327 ++++++++++++++++++
> lib/librte_telemetry/telemetry_data.h         |  46 +++
> 7 files changed, 480 insertions(+), 2 deletions(-)
> create mode 100644 lib/librte_telemetry/telemetry.c
> create mode 100644 lib/librte_telemetry/telemetry_data.h
> 
> diff --git a/lib/librte_telemetry/Makefile b/lib/librte_telemetry/Makefile
> index 2d7e442ab0..270e1aac54 100644
> --- a/lib/librte_telemetry/Makefile
> +++ b/lib/librte_telemetry/Makefile
> @@ -9,6 +9,9 @@ LIB = librte_telemetry.a
> CFLAGS += -O3
> CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)
> CFLAGS += -I$(RTE_SDK)/lib/librte_metrics/
> +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/include
> +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(ARCH_DIR)/include
> +CFLAGS += -pthread
> 
> LDLIBS += -lrte_eal
> LDLIBS += -lpthread
> @@ -20,6 +23,7 @@ EXPORT_MAP := rte_telemetry_version.map
> SRCS-$(CONFIG_RTE_LIBRTE_TELEMETRY) := rte_telemetry.c
> SRCS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += rte_telemetry_parser.c
> SRCS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += rte_telemetry_parser_test.c
> +SRCS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += telemetry.c
> 
> # export include files
> SYMLINK-$(CONFIG_RTE_LIBRTE_TELEMETRY)-include := rte_telemetry.h
> diff --git a/lib/librte_telemetry/meson.build b/lib/librte_telemetry/meson.build
> index 18b214a8e8..0cdae414a4 100644
> --- a/lib/librte_telemetry/meson.build
> +++ b/lib/librte_telemetry/meson.build
> @@ -1,7 +1,10 @@
> # SPDX-License-Identifier: BSD-3-Clause
> # Copyright(c) 2018 Intel Corporation
> 
> -sources = files('rte_telemetry.c', 'rte_telemetry_parser.c', 'rte_telemetry_parser_test.c')
> +includes = [global_inc]
> +
> +sources = files('rte_telemetry.c', 'rte_telemetry_parser.c', 'rte_telemetry_parser_test.c',
> +	'telemetry.c')
> headers = files('rte_telemetry.h', 'rte_telemetry_internal.h', 'rte_telemetry_parser.h')
> includes += include_directories('../librte_metrics')
> 
> diff --git a/lib/librte_telemetry/rte_telemetry.c b/lib/librte_telemetry/rte_telemetry.c
> index 2fb8ffe873..45b6d9d948 100644
> --- a/lib/librte_telemetry/rte_telemetry.c
> +++ b/lib/librte_telemetry/rte_telemetry.c
> @@ -503,6 +503,9 @@ rte_telemetry_init(void)
> 		return -EPERM;
> 	}
> 
> +	if (rte_telemetry_new_init() != 0)
> +		return -1;
> +
> 	return 0;
> }
> 
> diff --git a/lib/librte_telemetry/rte_telemetry.h b/lib/librte_telemetry/rte_telemetry.h
> index aedb318598..66290a3fdf 100644
> --- a/lib/librte_telemetry/rte_telemetry.h
> +++ b/lib/librte_telemetry/rte_telemetry.h
> @@ -3,19 +3,80 @@
>  */
> 
> #include <stdint.h>
> +#include <rte_compat.h>
> 
> #ifndef _RTE_TELEMETRY_H_
> #define _RTE_TELEMETRY_H_
> 
> +/** Maximum number of telemetry callbacks. */
> +#define TELEMETRY_MAX_CALLBACKS 64
> +/** Maximum length for string used in object. */
> +#define RTE_TEL_MAX_STRING_LEN 64
> +/** Maximum length of string. */
> +#define RTE_TEL_MAX_SINGLE_STRING_LEN 8192
> +/** Maximum number of dictionary entries. */
> +#define RTE_TEL_MAX_DICT_ENTRIES 256
> +/** Maximum number of array entries. */
> +#define RTE_TEL_MAX_ARRAY_ENTRIES 512
> +
> /**
> + * @warning
> + * @b EXPERIMENTAL: all functions in this file may change without prior notice
> + *
>  * @file
>  * RTE Telemetry
>  *
>  * The telemetry library provides a method to retrieve statistics from
> - * DPDK by sending a JSON encoded message over a socket. DPDK will send
> + * DPDK by sending a request message over a socket. DPDK will send
>  * a JSON encoded response containing telemetry data.
>  ***/
> 
> +/** opaque structure used internally for managing data from callbacks */
> +struct rte_tel_data;
> +
> +/**
> + * The types of data that can be managed in arrays or dicts.
> + * For arrays, this must be specified at creation time, while for
> + * dicts this is specified implicitly each time an element is added
> + * via calling a type-specific function.
> + */
> +enum rte_tel_value_type {
> +	RTE_TEL_STRING_VAL, /** a string value */
> +	RTE_TEL_INT_VAL,    /** a signed 32-bit int value */
> +	RTE_TEL_U64_VAL,    /** an unsigned 64-bit int value */
> +};
> +
> +/**
> + * This telemetry callback is used when registering a telemetry command.
> + * It handles getting and formatting information to be returned to telemetry
> + * when requested.
> + *
> + * @param cmd
> + * The cmd that was requested by the client.
> + * @param params
> + * Contains data required by the callback function.
> + * @param info
> + * The information to be returned to the caller.
> + *
> + * @return
> + * Length of buffer used on success.
> + * @return
> + * Negative integer on error.
> + */
> +typedef int (*telemetry_cb)(const char *cmd, const char *params,
> +		struct rte_tel_data *info);
> +
> +/**
> + * Used for handling data received over a telemetry socket.
> + *
> + * @param sock_id
> + * ID for the socket to be used by the handler.
> + *
> + * @return
> + * Void.
> + */
> +typedef void * (*handler)(void *sock_id);
> +
> /**
>  * @warning
>  * @b EXPERIMENTAL: this API may change without prior notice
> @@ -66,4 +127,36 @@ __rte_experimental
> int32_t
> rte_telemetry_selftest(void);
> 
> +/**
> + * Used when registering a command and callback function with telemetry.
> + *
> + * @param cmd
> + * The command to register with telemetry.
> + * @param fn
> + * Callback function to be called when the command is requested.
> + * @param help
> + * Help text for the command.
> + *
> + * @return
> + *  0 on success.
> + * @return
> + *  -EINVAL for invalid parameters failure.
> + *  @return
> + *  -ENOENT if max callbacks limit has been reached.
> + */
> +__rte_experimental
> +int
> +rte_telemetry_register_cmd(const char *cmd, telemetry_cb fn, const char *help);
> +
> +/**
> + * Initialize new version of Telemetry.
> + *
> + * @return
> + *  0 on success.
> + * @return
> + *  -1 on failure.
> + */
> +__rte_experimental
> +int
> +rte_telemetry_new_init(void);
> #endif
> diff --git a/lib/librte_telemetry/rte_telemetry_version.map b/lib/librte_telemetry/rte_telemetry_version.map
> index a80058c59c..831bbd59ad 100644
> --- a/lib/librte_telemetry/rte_telemetry_version.map
> +++ b/lib/librte_telemetry/rte_telemetry_version.map
> @@ -6,6 +6,8 @@ EXPERIMENTAL {
> 	rte_telemetry_parse;
> 	rte_telemetry_selftest;
> 	rte_telemetry_set_metrics_fns;
> +	rte_telemetry_new_init;
> +	rte_telemetry_register_cmd;
> 
> 	local: *;
> };
> diff --git a/lib/librte_telemetry/telemetry.c b/lib/librte_telemetry/telemetry.c
> new file mode 100644
> index 0000000000..b71915bb69
> --- /dev/null
> +++ b/lib/librte_telemetry/telemetry.c
> @@ -0,0 +1,327 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2020 Intel Corporation
> + */
> +
> +#include <unistd.h>
> +#include <pthread.h>
> +#include <sys/socket.h>
> +#include <sys/un.h>
> +#include <dlfcn.h>
> +
> +/* we won't link against libbsd, so just always use DPDKs-specific strlcpy */
> +#undef RTE_USE_LIBBSD
> +#include <rte_string_fns.h>
> +#include <rte_common.h>
> +#include <rte_spinlock.h>
> +#include <rte_version.h>
> +
> +#include "rte_telemetry.h"
> +#include "telemetry_json.h"
> +#include "telemetry_data.h"
> +
> +#define MAX_CMD_LEN 56
> +#define MAX_HELP_LEN 64
> +#define MAX_OUTPUT_LEN (1024 * 16)
> +
> +static void *
> +client_handler(void *socket);
> +
> +struct cmd_callback {
> +	char cmd[MAX_CMD_LEN];
> +	telemetry_cb fn;
> +	char help[MAX_HELP_LEN];
> +};
> +
> +struct socket {
> +	int sock;
> +	char path[sizeof(((struct sockaddr_un *)0)->sun_path)];
> +	handler fn;
> +};
> +static struct socket v2_socket; /* socket for v2 telemetry */
> +static char telemetry_log_error[1024]; /* Will contain error on init failure */
> +/* list of command callbacks, with one command registered by default */
> +static struct cmd_callback callbacks[TELEMETRY_MAX_CALLBACKS];
> +static int num_callbacks; /* How many commands are registered */
> +/* Used when accessing or modifying list of command callbacks */
> +static rte_spinlock_t callback_sl = RTE_SPINLOCK_INITIALIZER;
> +
> +int
> +rte_telemetry_register_cmd(const char *cmd, telemetry_cb fn, const char *help)
> +{
> +	int i = 0;
> +
> +	if (strlen(cmd) >= MAX_CMD_LEN || fn == NULL || cmd[0] != '/'
> +			|| strlen(help) >= MAX_HELP_LEN)
> +		return -EINVAL;
> +	if (num_callbacks >= TELEMETRY_MAX_CALLBACKS)
> +		return -ENOENT;
> +
> +	rte_spinlock_lock(&callback_sl);
> +	while (i < num_callbacks && strcmp(cmd, callbacks[i].cmd) > 0)
> +		i++;
> +	if (i != num_callbacks)
> +		/* Move elements to keep the list alphabetical */
> +		memmove(callbacks + i + 1, callbacks + i,
> +			sizeof(struct cmd_callback) * (num_callbacks - i));
> +
> +	strlcpy(callbacks[i].cmd, cmd, MAX_CMD_LEN);
> +	callbacks[i].fn = fn;
> +	strlcpy(callbacks[i].help, help, MAX_HELP_LEN);
> +	num_callbacks++;
> +	rte_spinlock_unlock(&callback_sl);
> +
> +	return 0;
> +}
> +
> +static void
> +output_json(const char *cmd, const struct rte_tel_data *d, int s)
> +{
> +	char out_buf[MAX_OUTPUT_LEN];
> +
> +	char *cb_data_buf;
> +	size_t buf_len, prefix_used, used = 0;
> +	unsigned int i;
> +
> +	RTE_BUILD_BUG_ON(sizeof(out_buf) < MAX_CMD_LEN +
> +			RTE_TEL_MAX_SINGLE_STRING_LEN + 10);
> +	switch (d->type) {
> +	case RTE_TEL_NULL:
> +		used = snprintf(out_buf, sizeof(out_buf), "{\"%.*s\":null}",
> +				MAX_CMD_LEN, cmd);
> +		break;
> +	case RTE_TEL_STRING:
> +		used = snprintf(out_buf, sizeof(out_buf), "{\"%.*s\":\"%.*s\"}",
> +				MAX_CMD_LEN, cmd,
> +				RTE_TEL_MAX_SINGLE_STRING_LEN, d->data.str);
> +		break;
> +	case RTE_TEL_DICT:
> +		prefix_used = snprintf(out_buf, sizeof(out_buf), "{\"%.*s\":",
> +				MAX_CMD_LEN, cmd);
> +		cb_data_buf = &out_buf[prefix_used];
> +		buf_len = sizeof(out_buf) - prefix_used - 1; /* space for '}' */
> +
> +		used = rte_tel_json_empty_obj(cb_data_buf, buf_len, 0);
> +		for (i = 0; i < d->data_len; i++) {
> +			const struct tel_dict_entry *v = &d->data.dict[i];
> +			switch (v->type) {
> +			case RTE_TEL_STRING_VAL:
> +				used = rte_tel_json_add_obj_str(cb_data_buf,
> +						buf_len, used,
> +						v->name, v->value.sval);
> +				break;
> +			case RTE_TEL_INT_VAL:
> +				used = rte_tel_json_add_obj_int(cb_data_buf,
> +						buf_len, used,
> +						v->name, v->value.ival);
> +				break;
> +			case RTE_TEL_U64_VAL:
> +				used = rte_tel_json_add_obj_u64(cb_data_buf,
> +						buf_len, used,
> +						v->name, v->value.u64val);
> +				break;
> +			}
> +		}
> +		used += prefix_used;
> +		used += strlcat(out_buf + used, "}", sizeof(out_buf) - used);
> +		break;
> +	case RTE_TEL_ARRAY_STRING:
> +	case RTE_TEL_ARRAY_INT:
> +	case RTE_TEL_ARRAY_U64:
> +		prefix_used = snprintf(out_buf, sizeof(out_buf), "{\"%.*s\":",
> +				MAX_CMD_LEN, cmd);
> +		cb_data_buf = &out_buf[prefix_used];
> +		buf_len = sizeof(out_buf) - prefix_used - 1; /* space for '}' */
> +
> +		used = rte_tel_json_empty_array(cb_data_buf, buf_len, 0);
> +		for (i = 0; i < d->data_len; i++)
> +			if (d->type == RTE_TEL_ARRAY_STRING)
> +				used = rte_tel_json_add_array_string(
> +						cb_data_buf,
> +						buf_len, used,
> +						d->data.array[i].sval);
> +			else if (d->type == RTE_TEL_ARRAY_INT)
> +				used = rte_tel_json_add_array_int(cb_data_buf,
> +						buf_len, used,
> +						d->data.array[i].ival);
> +			else if (d->type == RTE_TEL_ARRAY_U64)
> +				used = rte_tel_json_add_array_u64(cb_data_buf,
> +						buf_len, used,
> +						d->data.array[i].u64val);
> +		used += prefix_used;
> +		used += strlcat(out_buf + used, "}", sizeof(out_buf) - used);
> +		break;
> +	}
> +	if (write(s, out_buf, used) < 0)
> +		perror("Error writing to socket");
> +}
> +
> +static void
> +perform_command(telemetry_cb fn, const char *cmd, const char *param, int s)
> +{
> +	struct rte_tel_data data;
> +
> +	int ret = fn(cmd, param, &data);
> +	if (ret < 0) {
> +		char out_buf[MAX_CMD_LEN + 10];
> +		int used = snprintf(out_buf, sizeof(out_buf), "{\"%.*s\":null}",
> +				MAX_CMD_LEN, cmd ?: "none");
> +		if (write(s, out_buf, used) < 0)
> +			perror("Error writing to socket");
> +		return;
> +	}
> +	output_json(cmd, &data, s);
> +}
> +
> +static int
> +unknown_command(const char *cmd __rte_unused, const char *params __rte_unused,
> +		struct rte_tel_data *d)
> +{
> +	return d->type = RTE_TEL_NULL;
> +}
> +
> +static void *
> +client_handler(void *sock_id)
> +{
> +	int s = (int)(uintptr_t)sock_id;
> +	char buffer[1024];
> +	char info_str[1024];
> +	snprintf(info_str, sizeof(info_str),
> +			"{\"version\":\"%s\",\"pid\":%d,\"max_output_len\":%d}",
> +			rte_version(), getpid(), MAX_OUTPUT_LEN);
> +	if (write(s, info_str, strlen(info_str)) < 0) {
> +		close(s);
> +		return NULL;
> +	}
> +
> +	/* receive data is not null terminated */
> +	int bytes = read(s, buffer, sizeof(buffer) - 1);
> +	buffer[bytes] = 0;

Bytes could be a negative number from read returning an error.

> +	while (bytes > 0) {
> +		const char *cmd = strtok(buffer, ",");
> +		const char *param = strtok(NULL, ",");
> +		telemetry_cb fn = unknown_command;
> +		int i;
> +
> +		if (cmd && strlen(cmd) < MAX_CMD_LEN) {
> +			rte_spinlock_lock(&callback_sl);
> +			for (i = 0; i < num_callbacks; i++)
> +				if (strcmp(cmd, callbacks[i].cmd) == 0) {
> +					fn = callbacks[i].fn;
> +					break;
> +				}
> +			rte_spinlock_unlock(&callback_sl);
> +		}
> +		perform_command(fn, cmd, param, s);
> +
> +		bytes = read(s, buffer, sizeof(buffer) - 1);
> +		buffer[bytes] = 0;

Same here.
> +	}
> +	close(s);
> +	return NULL;
> +}
> +
> +static void *
> +socket_listener(void *socket)
> +{
> +	while (1) {
> +		pthread_t th;
> +		struct socket *s = (struct socket *)socket;
> +		int s_accepted = accept(s->sock, NULL, NULL);
> +		if (s_accepted < 0) {
> +			snprintf(telemetry_log_error,
> +					sizeof(telemetry_log_error),
> +					"Error with accept, telemetry thread quitting\n");
> +			return NULL;
> +		}
> +		pthread_create(&th, NULL, s->fn, (void *)(uintptr_t)s_accepted);
> +		pthread_detach(th);
> +	}
> +	return NULL;
> +}
> +
> +static inline char *
> +get_socket_path(const char *runtime_dir, const int version)
> +{
> +	static char path[PATH_MAX];
> +	snprintf(path, sizeof(path), "%s/dpdk_telemetry.v%d",
> +			strlen(runtime_dir) ? runtime_dir : "/tmp", version);
> +	return path;
> +}
> +
> +static void
> +unlink_sockets(void)
> +{
> +	if (v2_socket.path[0])
> +		unlink(v2_socket.path);
> +}
> +
> +static int
> +create_socket(char *path)
> +{
> +	int sock = socket(AF_UNIX, SOCK_SEQPACKET, 0);
> +	if (sock < 0) {
> +		snprintf(telemetry_log_error, sizeof(telemetry_log_error),
> +				"Error with socket creation, %s",
> +				strerror(errno));
> +		return -1;
> +	}
> +
> +	struct sockaddr_un sun = {.sun_family = AF_UNIX};
> +	strlcpy(sun.sun_path, path, sizeof(sun.sun_path));
> +	unlink(sun.sun_path);
> +	if (bind(sock, (void *) &sun, sizeof(sun)) < 0) {
> +		snprintf(telemetry_log_error, sizeof(telemetry_log_error),
> +				"Error binding socket: %s",
> +				strerror(errno));
> +		sun.sun_path[0] = 0;
> +		goto error;
> +	}
> +
> +	if (listen(sock, 1) < 0) {
> +		snprintf(telemetry_log_error, sizeof(telemetry_log_error),
> +				"Error calling listen for socket: %s",
> +				strerror(errno));
> +		goto error;
> +	}
> +
> +	return sock;
> +
> +error:
> +	close(sock);
> +	unlink_sockets();
> +	return -1;
> +}
> +
> +static int
> +telemetry_v2_init(const char *runtime_dir)
> +{
> +	pthread_t t_new;
> +
> +	v2_socket.fn = client_handler;
> +	if (strlcpy(v2_socket.path, get_socket_path(runtime_dir, 2),
> +			sizeof(v2_socket.path)) >= sizeof(v2_socket.path)) {
> +		snprintf(telemetry_log_error, sizeof(telemetry_log_error),
> +				"Error with socket binding, path too long");
> +		return -1;
> +	}
> +
> +	v2_socket.sock = create_socket(v2_socket.path);
> +	if (v2_socket.sock < 0)
> +		return -1;
> +	pthread_create(&t_new, NULL, socket_listener, &v2_socket);
> +	atexit(unlink_sockets);
> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_telemetry_new_init(void)
> +{
> +	const char *error_str;
> +	if (telemetry_v2_init(rte_eal_get_runtime_dir()) != 0) {
> +		error_str = telemetry_log_error;
> +		printf("Error initialising telemetry - %s", error_str);
> +		return -1;
> +	}
> +	return 0;
> +}
> diff --git a/lib/librte_telemetry/telemetry_data.h b/lib/librte_telemetry/telemetry_data.h
> new file mode 100644
> index 0000000000..ff3a371a33
> --- /dev/null
> +++ b/lib/librte_telemetry/telemetry_data.h
> @@ -0,0 +1,46 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2020 Intel Corporation
> + */
> +
> +#ifndef _TELEMETRY_DATA_H_
> +#define _TELEMETRY_DATA_H_
> +
> +#include <inttypes.h>
> +#include "rte_telemetry.h"
> +
> +enum tel_container_types {
> +	RTE_TEL_NULL,	      /** null, used as error value */
> +	RTE_TEL_STRING,	      /** basic string type, no included data */
> +	RTE_TEL_DICT,	      /** name-value pairs, of individual value type */
> +	RTE_TEL_ARRAY_STRING, /** array of string values only */
> +	RTE_TEL_ARRAY_INT,    /** array of signed, 32-bit int values */
> +	RTE_TEL_ARRAY_U64,    /** array of unsigned 64-bit int values */
> +};
> +
> +/* each type here must have an equivalent enum in the value types enum in
> + * telemetry.h and an array type defined above, and have appropriate
> + * type assignment in the RTE_TEL_data_start_array() function
> + */
> +union tel_value {
> +	char sval[RTE_TEL_MAX_STRING_LEN];
> +	int ival;
> +	uint64_t u64val;
> +};
> +
> +struct tel_dict_entry {
> +	char name[RTE_TEL_MAX_STRING_LEN];
> +	enum rte_tel_value_type type;
> +	union tel_value value;
> +};
> +
> +struct rte_tel_data {
> +	enum tel_container_types type;
> +	unsigned int data_len; /* for array or object, how many items */
> +	union {
> +		char str[RTE_TEL_MAX_SINGLE_STRING_LEN];
> +		struct tel_dict_entry dict[RTE_TEL_MAX_DICT_ENTRIES];
> +		union tel_value array[RTE_TEL_MAX_ARRAY_ENTRIES];
> +	} data; /* data container */
> +};
> +
> +#endif
> -- 
> 2.17.1
>