[dpdk-dev] [PATCH v1 2/2] examples/vdpa: add a new sample for vdpa

[Ye Xiaolong]

On 09/10, Wang, Haiyue wrote:
>Can we add options to support 'int rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)' API calling
>to get the device id ? Not just hard code 0 ~ num ?  Like  choosing one of vDPA engines explicitly by BDF: 
>-w 0000:06:00.2,vdpa=1 -w 0000:06:00.3,vdpa=1
>

Good suggestion, I'll adopt it in v2.

Thanks,
Xiaolong
>BR,
>Haiyue
>
>> -----Original Message-----
>> From: dev [mailto:dev-bounces at dpdk.org] On Behalf Of Xiaolong Ye
>> Sent: Thursday, September 6, 2018 21:17
>> To: dev at dpdk.org; Maxime Coquelin <maxime.coquelin at redhat.com>; Bie,
>> Tiwei <tiwei.bie at intel.com>; Wang, Zhihong <zhihong.wang at intel.com>
>> Cc: Wang, Xiao W <xiao.w.wang at intel.com>; Ye, Xiaolong
>> <xiaolong.ye at intel.com>
>> Subject: [dpdk-dev] [PATCH v1 2/2] examples/vdpa: add a new sample for vdpa
>> 
>> From: Xiao Wang <xiao.w.wang at intel.com>
>> 
>> This patch adds a sample which creates vhost-user socket based on vdpa driver.
>> vdpa driver can help to set up vhost datapath so this app doesn't need to spend a
>> dedicated worker thread on vhost enqueue/dequeue operations.
>> 
>> Below are setup steps for your reference (using IFC driver):
>> 
>> 1. Make sure your kernnel vhost module and QEMU support vhost-user.
>>    - OS: CentOS 7.4
>>    - QEMU: 2.10.1
>>    - Guest OS: CentOS 7.2
>> 
>> 2. Bind VFIO-pci to VF
>>    a) modprobe vfio-pci
>>    d) ./usertools/dpdk-devbind.py -b vfio-pci 06:00.2 06:00.3
>> 
>> 3. Start vdpa sample
>>     ./examples/vdpa/build/vdpa --log-level=9 -c 0x10 -n 4 --socket-mem
>>     1024,1024  -w 0000:06:00.2,vdpa=1 -w 0000:06:00.3,vdpa=1   --
>>     --iface  /tmp/vhost-user- (or use interactive mode to create vdpa
>>     ports)
>> 
>> 4. Start two VMs
>>    ./qemu-2.10.1/x86_64-softmmu/qemu-system-x86_64 -cpu host -enable-kvm
>> \
>>    <snip>
>>    -mem-prealloc \
>>    -chardev socket,id=char0,path=/tmp/vhost-user-0 \
>>    -netdev type=vhost-user,id=vdpa,chardev=char0,vhostforce \
>>    -device virtio-net-pci,netdev=vdpa,mac=00:aa:bb:cc:dd:ee \
>> 
>>    ./qemu-2.10.1/x86_64-softmmu/qemu-system-x86_64 -cpu host -enable-kvm
>> \
>>    <snip>
>>    -mem-prealloc \
>>    -chardev socket,id=char0,path=/tmp/vhost-user-1 \
>>    -netdev type=vhost-user,id=vdpa,chardev=char0,vhostforce \
>>    -device virtio-net-pci,netdev=vdpa,mac=00:aa:bb:cc:dd:ff \
>> 
>> 5. Login two VMs and configure the ip, verify the network connection via
>>    ping.
>> 
>> Signed-off-by: Xiao Wang <xiao.w.wang at intel.com>
>> Signed-off-by: Xiaolong Ye <xiaolong.ye at intel.com>
>> ---
>>  examples/vdpa/Makefile    |  32 ++++
>>  examples/vdpa/main.c      | 327 ++++++++++++++++++++++++++++++++++++++
>>  examples/vdpa/meson.build |  11 ++
>>  3 files changed, 370 insertions(+)
>>  create mode 100644 examples/vdpa/Makefile  create mode 100644
>> examples/vdpa/main.c  create mode 100644 examples/vdpa/meson.build
>> 
>> diff --git a/examples/vdpa/Makefile b/examples/vdpa/Makefile new file mode
>> 100644 index 000000000..42672a2bc
>> --- /dev/null
>> +++ b/examples/vdpa/Makefile
>> @@ -0,0 +1,32 @@
>> +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
>> +Corporation
>> +
>> +ifeq ($(RTE_SDK),)
>> +$(error "Please define RTE_SDK environment variable") endif
>> +
>> +# Default target, can be overridden by command line or environment
>> +RTE_TARGET ?= x86_64-native-linuxapp-gcc
>> +
>> +include $(RTE_SDK)/mk/rte.vars.mk
>> +
>> +ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
>> +$(info This application can only operate in a linuxapp environment, \
>> +please change the definition of the RTE_TARGET environment variable)
>> +all:
>> +else
>> +
>> +# binary name
>> +APP = vdpa
>> +
>> +# all source are stored in SRCS-y
>> +SRCS-y := main.c
>> +
>> +CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
>> +CFLAGS += $(WERROR_FLAGS)
>> +CFLAGS += -D_GNU_SOURCE
>> +CFLAGS += -DALLOW_EXPERIMENTAL_API
>> +
>> +include $(RTE_SDK)/mk/rte.extapp.mk
>> +
>> +endif
>> diff --git a/examples/vdpa/main.c b/examples/vdpa/main.c new file mode
>> 100644 index 000000000..b19d7a217
>> --- /dev/null
>> +++ b/examples/vdpa/main.c
>> @@ -0,0 +1,327 @@
>> +/* SPDX-License-Identifier: BSD-3-Clause
>> + * Copyright(c) 2018 Intel Corporation
>> + */
>> +
>> +#include <getopt.h>
>> +#include <signal.h>
>> +#include <stdint.h>
>> +#include <string.h>
>> +#include <unistd.h>
>> +
>> +#include <rte_ethdev.h>
>> +#include <rte_malloc.h>
>> +#include <rte_vhost.h>
>> +#include <rte_vdpa.h>
>> +
>> +#include <cmdline_parse.h>
>> +#include <cmdline_socket.h>
>> +#include <cmdline_parse_string.h>
>> +#include <cmdline.h>
>> +
>> +#define MAX_PATH_LEN 128
>> +#define MAX_VDPA_SAMPLE_PORTS 1024
>> +
>> +struct vdpa_port {
>> +	char ifname[MAX_PATH_LEN];
>> +	int did;
>> +	int vid;
>> +};
>> +
>> +struct vdpa_port vports[MAX_VDPA_SAMPLE_PORTS];
>> +
>> +char iface[MAX_PATH_LEN];
>> +int devcnt;
>> +int dev_id;
>> +uint8_t interactive;
>> +
>> +/* display usage */
>> +static void
>> +vdpa_usage(const char *prgname)
>> +{
>> +	printf("Usage: %s [EAL options] -- "
>> +				 "[--interactive|-i] "
>> +				 "[--iface PATH]\n"
>> +				 "	--interactive: run in interactive
>> mode.\n"
>> +				 "	--iface <path>: specify the path prefix
>> of the socket files, e.g. /tmp/vhost-user-\n",
>> +				 prgname);
>> +}
>> +
>> +static int
>> +parse_args(int argc, char **argv)
>> +{
>> +	static const char *short_option = "i";
>> +	static struct option long_option[] = {
>> +		{"iface", required_argument, NULL, 0},
>> +		{"interactive", 0, 0, 0},
>> +		{NULL, 0, 0, 0},
>> +	};
>> +	int opt, idx;
>> +	char *prgname = argv[0];
>> +
>> +	while ((opt = getopt_long(argc, argv, short_option, long_option, &idx))
>> +			!= EOF) {
>> +		switch (opt) {
>> +		case 'i':
>> +			printf("Interactive-mode selected\n");
>> +			interactive = 1;
>> +			break;
>> +		/* long options */
>> +		case 0:
>> +			if (strncmp(long_option[idx].name, "iface",
>> +						MAX_PATH_LEN) == 0) {
>> +				strncpy(iface, optarg, MAX_PATH_LEN);
>> +				printf("iface %s\n", iface);
>> +			}
>> +			if (!strcmp(long_option[idx].name, "interactive")) {
>> +				printf("Interactive-mode selected\n");
>> +				interactive = 1;
>> +			}
>> +			break;
>> +
>> +		default:
>> +			vdpa_usage(prgname);
>> +			return -1;
>> +		}
>> +	}
>> +
>> +	if (*iface == '\0' && interactive == 0) {
>> +		vdpa_usage(prgname);
>> +		return -1;
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +static int
>> +data_init(void)
>> +{
>> +	devcnt = rte_vdpa_get_device_num();
>> +	if (devcnt <= 0) {
>> +		printf("No available vdpa device found\n");
>> +		return -1;
>> +	}
>> +	dev_id = 0;
>> +	interactive = 0;
>> +	memset(iface, 0, MAX_PATH_LEN * sizeof(iface[0]));
>> +	memset(vports, 0, MAX_VDPA_SAMPLE_PORTS * sizeof(vports[0]));
>> +	return 0;
>> +}
>> +
>> +static void
>> +signal_handler(int signum)
>> +{
>> +	if (signum == SIGINT || signum == SIGTERM) {
>> +		printf("\nSignal %d received, preparing to exit...\n",
>> +				signum);
>> +		exit(0);
>> +	}
>> +}
>> +
>> +static int
>> +new_device(int vid)
>> +{
>> +	char ifname[MAX_PATH_LEN];
>> +	int i;
>> +
>> +	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
>> +	for (i = 0; i < MAX_VDPA_SAMPLE_PORTS; i++) {
>> +		if (strncmp(ifname, vports[i].ifname, MAX_PATH_LEN) == 0) {
>> +			printf("\nnew port %s, did: %d\n",
>> +					ifname, vports[i].did);
>> +			vports[i].vid = vid;
>> +			break;
>> +		}
>> +	}
>> +
>> +	if (i >= MAX_VDPA_SAMPLE_PORTS)
>> +		return -1;
>> +
>> +	return 0;
>> +}
>> +
>> +static void
>> +destroy_device(int vid)
>> +{
>> +	char ifname[MAX_PATH_LEN];
>> +	int i;
>> +
>> +	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
>> +	for (i = 0; i < MAX_VDPA_SAMPLE_PORTS; i++) {
>> +		if (strcmp(ifname, vports[i].ifname) == 0) {
>> +			printf("\ndestroy port %s, did: %d\n",
>> +					ifname, vports[i].did);
>> +			break;
>> +		}
>> +	}
>> +}
>> +
>> +static const struct vhost_device_ops vdpa_sample_devops = {
>> +	.new_device = new_device,
>> +	.destroy_device = destroy_device,
>> +	.vring_state_changed = NULL,
>> +	.features_changed = NULL,
>> +	.new_connection = NULL,
>> +	.destroy_connection = NULL,
>> +};
>> +
>> +static void
>> +start_vdpa(const char *socket_path, int did, uint64_t flags) {
>> +	int ret;
>> +	ret = rte_vhost_driver_register(socket_path, flags);
>> +	if (ret != 0)
>> +		rte_exit(EXIT_FAILURE,
>> +			"register driver failed: %s\n",
>> +			socket_path);
>> +
>> +	ret = rte_vhost_driver_callback_register(socket_path,
>> +			&vdpa_sample_devops);
>> +	if (ret != 0)
>> +		rte_exit(EXIT_FAILURE,
>> +			"register driver ops failed: %s\n",
>> +			socket_path);
>> +
>> +	ret = rte_vhost_driver_attach_vdpa_device(socket_path, did);
>> +	if (ret != 0)
>> +		rte_exit(EXIT_FAILURE,
>> +			"attach vdpa device failed: %s\n",
>> +			socket_path);
>> +
>> +	if (rte_vhost_driver_start(socket_path) < 0)
>> +		rte_exit(EXIT_FAILURE,
>> +			"start vhost driver failed: %s\n",
>> +			socket_path);
>> +}
>> +
>> +/* interactive cmds */
>> +
>> +/* *** Help command with introduction. *** */ struct cmd_help_result {
>> +	cmdline_fixed_string_t help;
>> +};
>> +
>> +static void cmd_help_parsed(__attribute__((unused)) void *parsed_result,
>> +				struct cmdline *cl,
>> +				__attribute__((unused)) void *data) {
>> +	cmdline_printf(
>> +		cl,
>> +		"\n"
>> +		"The following commands are currently available:\n\n"
>> +		"Control:\n"
>> +		"    help                                      : Show interactive instructions.\n"
>> +		"    create <socket_file_path>                 : Create a new vdpa
>> port.\n\n"
>> +	);
>> +}
>> +
>> +cmdline_parse_token_string_t cmd_help_help =
>> +	TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
>> +
>> +cmdline_parse_inst_t cmd_help = {
>> +	.f = cmd_help_parsed,
>> +	.data = NULL,
>> +	.help_str = "show help",
>> +	.tokens = {
>> +		(void *)&cmd_help_help,
>> +		NULL,
>> +	},
>> +};
>> +
>> +/* *** Create new vdpa port *** */
>> +struct cmd_create_result {
>> +	cmdline_fixed_string_t action;
>> +	cmdline_fixed_string_t socket_path;
>> +};
>> +
>> +static void cmd_create_vdpa_port_parsed(void *parsed_result,
>> +				struct cmdline *cl,
>> +				__attribute__((unused)) void *data) {
>> +	struct cmd_create_result *res = parsed_result;
>> +	char ifname[MAX_PATH_LEN];
>> +	uint64_t flags = 0;
>> +
>> +	strncpy(ifname, res->socket_path, MAX_PATH_LEN);
>> +	if (dev_id < devcnt)
>> +		start_vdpa(ifname, dev_id++, flags);
>> +	else
>> +		cmdline_printf(cl, "No available vdpa devices.\n"); }
>> +
>> +cmdline_parse_token_string_t cmd_action_create =
>> +	TOKEN_STRING_INITIALIZER(struct cmd_create_result, action, "create");
>> +cmdline_parse_token_string_t cmd_socket_path =
>> +	TOKEN_STRING_INITIALIZER(struct cmd_create_result, socket_path,
>> NULL);
>> +
>> +cmdline_parse_inst_t cmd_create_vdpa_port = {
>> +	.f = cmd_create_vdpa_port_parsed,  /* function to call */
>> +	.data = NULL,      /* 2nd arg of func */
>> +	.help_str = "Create an new vdpa port",
>> +	.tokens = {        /* token list, NULL terminated */
>> +		(void *)&cmd_action_create,
>> +		(void *)&cmd_socket_path,
>> +		NULL,
>> +	},
>> +};
>> +
>> +cmdline_parse_ctx_t main_ctx[] = {
>> +	(cmdline_parse_inst_t *)&cmd_help,
>> +	(cmdline_parse_inst_t *)&cmd_create_vdpa_port,
>> +	NULL,
>> +};
>> +
>> +int
>> +main(int argc, char *argv[])
>> +{
>> +	char ifname[MAX_PATH_LEN];
>> +	char ch;
>> +	int i, did;
>> +	int ret;
>> +	uint64_t flags = 0;
>> +	struct cmdline *cl;
>> +
>> +	ret = rte_eal_init(argc, argv);
>> +	if (ret < 0)
>> +		rte_exit(EXIT_FAILURE, "eal init failed\n");
>> +	argc -= ret;
>> +	argv += ret;
>> +
>> +	signal(SIGINT, signal_handler);
>> +	signal(SIGTERM, signal_handler);
>> +
>> +	data_init();
>> +
>> +	ret = parse_args(argc, argv);
>> +	if (ret < 0)
>> +		rte_exit(EXIT_FAILURE, "invalid argument\n");
>> +
>> +	if (interactive == 1) {
>> +		cl = cmdline_stdin_new(main_ctx, "vdpa> ");
>> +		if (cl == NULL)
>> +			rte_panic("Cannot create cmdline instance\n");
>> +		cmdline_interact(cl);
>> +		cmdline_stdin_exit(cl);
>> +	} else {
>> +		for (i = 0; i <  RTE_MIN(MAX_VDPA_SAMPLE_PORTS, devcnt);
>> i++) {
>> +			did = i;
>> +			snprintf(ifname, sizeof(ifname), "%s%d", iface, i);
>> +			vports[i].did = did;
>> +			strncpy(vports[i].ifname, ifname, MAX_PATH_LEN);
>> +
>> +			start_vdpa(ifname, did, flags);
>> +
>> +			did++;
>> +		}
>> +
>> +		printf("enter \'q\' to quit\n");
>> +		while (scanf("%c", &ch)) {
>> +			if (ch == 'q')
>> +				break;
>> +			while (ch != '\n')
>> +				scanf("%c", &ch);
>> +			printf("enter \'q\' to quit\n");
>> +		}
>> +	}
>> +
>> +	return 0;
>> +}
>> diff --git a/examples/vdpa/meson.build b/examples/vdpa/meson.build new file
>> mode 100644 index 000000000..e85d84a76
>> --- /dev/null
>> +++ b/examples/vdpa/meson.build
>> @@ -0,0 +1,11 @@
>> +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
>> +Corporation
>> +
>> +# meson file, for building this example as part of a main DPDK build.
>> +#
>> +# To build this example as a standalone application with an
>> +already-installed # DPDK instance, use 'make'
>> +
>> +sources = files(
>> +	'main.c'
>> +)
>> --
>> 2.18.0.rc1.1.g6f333ff2f
>