[dpdk-dev] [PATCH v4 05/12] virtio, qtest: Add QTest utility basic functions

[Tetsuya Mukawa]

The patch adds basic functions for accessing to QEMU quest that runs in
QTest mode. The functions will be used by virtio container extension
that can access to the above guest.

Signed-off-by: Tetsuya Mukawa <mukawa at igel.co.jp>
---
 config/common_base               |   1 +
 drivers/net/virtio/Makefile      |   4 +
 drivers/net/virtio/qtest_utils.c | 480 +++++++++++++++++++++++++++++++++++++++
 drivers/net/virtio/qtest_utils.h | 119 ++++++++++
 4 files changed, 604 insertions(+)
 create mode 100644 drivers/net/virtio/qtest_utils.c
 create mode 100644 drivers/net/virtio/qtest_utils.h

diff --git a/config/common_base b/config/common_base
index 340feaf..b19cb59 100644
--- a/config/common_base
+++ b/config/common_base
@@ -260,6 +260,7 @@ CONFIG_RTE_LIBRTE_VIRTIO_DEBUG_DUMP=n
 # Enable virtio support for container
 #
 CONFIG_RTE_VIRTIO_VDEV=n
+CONFIG_RTE_VIRTIO_VDEV_QTEST=n
 
 #
 # Compile burst-oriented VMXNET3 PMD driver
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
index 9e83852..e6d5a04 100644
--- a/drivers/net/virtio/Makefile
+++ b/drivers/net/virtio/Makefile
@@ -59,6 +59,10 @@ ifeq ($(CONFIG_RTE_VIRTIO_VDEV),y)
 	SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += vhost_embedded.c
 endif
 
+ifeq ($(CONFIG_RTE_VIRTIO_VDEV_QTEST),y)
+	SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += qtest_utils.c
+endif
+
 # this lib depends upon:
 DEPDIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += lib/librte_eal lib/librte_ether
 DEPDIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += lib/librte_mempool lib/librte_mbuf
diff --git a/drivers/net/virtio/qtest_utils.c b/drivers/net/virtio/qtest_utils.c
new file mode 100644
index 0000000..f4cd6af
--- /dev/null
+++ b/drivers/net/virtio/qtest_utils.c
@@ -0,0 +1,480 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 IGEL Co., Ltd. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of IGEL Co., Ltd. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <unistd.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <pthread.h>
+#include <fcntl.h>
+
+#include <rte_malloc.h>
+
+#include "virtio_logs.h"
+#include "virtio_ethdev.h"
+#include "qtest_utils.h"
+
+union qtest_pipefds {
+	struct {
+		int pipefd[2];
+	};
+	struct {
+		int readfd;
+		int writefd;
+	};
+};
+
+struct qtest_session {
+	int qtest_socket;
+	pthread_mutex_t qtest_session_lock;
+
+	pthread_t event_th;
+	int event_th_started;
+	char *evq;
+	char *evq_dequeue_ptr;
+	size_t evq_total_len;
+
+	union qtest_pipefds msgfds;
+};
+
+static int
+qtest_raw_send(int fd, char *buf, size_t count)
+{
+	size_t len = count;
+	size_t total_len = 0;
+	int ret = 0;
+
+	while (len > 0) {
+		ret = write(fd, buf, len);
+		if (ret == -1) {
+			if (errno == EINTR)
+				continue;
+			return ret;
+		}
+		if (ret == (int)len)
+			break;
+		total_len += ret;
+		buf += ret;
+		len -= ret;
+	}
+	return total_len + ret;
+}
+
+static int
+qtest_raw_recv(int fd, char *buf, size_t count)
+{
+	size_t len = count;
+	size_t total_len = 0;
+	int ret = 0;
+
+	while (len > 0) {
+		ret = read(fd, buf, len);
+		if (ret <= 0) {
+			if (errno == EINTR) {
+				continue;
+			}
+			return ret;
+		}
+		if (ret == (int)len)
+			break;
+		if (*(buf + ret - 1) == '\n')
+			break;
+		total_len += ret;
+		buf += ret;
+		len -= ret;
+	}
+	return total_len + ret;
+}
+
+/*
+ * To know QTest protocol specification, see below QEMU source code.
+ *  - qemu/qtest.c
+ * If qtest socket is closed, qtest_raw_in and qtest_raw_read will return 0.
+ */
+static uint32_t
+qtest_raw_in(struct qtest_session *s, uint16_t addr, char type)
+{
+	char buf[64];
+	int ret;
+
+	if ((type != 'l') && (type != 'w') && (type != 'b'))
+		rte_panic("Invalid value\n");
+
+	snprintf(buf, sizeof(buf), "in%c 0x%x\n", type, addr);
+	/* write to qtest socket */
+	ret = qtest_raw_send(s->qtest_socket, buf, strlen(buf));
+	/* read reply from event handler */
+	ret = qtest_raw_recv(s->msgfds.readfd, buf, sizeof(buf));
+	if (ret < 0)
+		return 0;
+
+	buf[ret] = '\0';
+	return strtoul(buf + strlen("OK "), NULL, 16);
+}
+
+static void
+qtest_raw_out(struct qtest_session *s, uint16_t addr, uint32_t val, char type)
+{
+	char buf[64];
+	int ret __rte_unused;
+
+	if ((type != 'l') && (type != 'w') && (type != 'b'))
+		rte_panic("Invalid value\n");
+
+	snprintf(buf, sizeof(buf), "out%c 0x%x 0x%x\n", type, addr, val);
+	/* write to qtest socket */
+	ret = qtest_raw_send(s->qtest_socket, buf, strlen(buf));
+	/* read reply from event handler */
+	ret = qtest_raw_recv(s->msgfds.readfd, buf, sizeof(buf));
+}
+
+static uint32_t
+qtest_raw_read(struct qtest_session *s, uint64_t addr, char type)
+{
+	char buf[64];
+	int ret;
+
+	if ((type != 'l') && (type != 'w') && (type != 'b'))
+		rte_panic("Invalid value\n");
+
+	snprintf(buf, sizeof(buf), "read%c 0x%lx\n", type, addr);
+	/* write to qtest socket */
+	ret = qtest_raw_send(s->qtest_socket, buf, strlen(buf));
+	/* read reply from event handler */
+	ret = qtest_raw_recv(s->msgfds.readfd, buf, sizeof(buf));
+	if (ret < 0)
+		return 0;
+
+	buf[ret] = '\0';
+	return strtoul(buf + strlen("OK "), NULL, 16);
+}
+
+static void
+qtest_raw_write(struct qtest_session *s, uint64_t addr, uint32_t val, char type)
+{
+	char buf[64];
+	int ret __rte_unused;
+
+	if ((type != 'l') && (type != 'w') && (type != 'b'))
+		rte_panic("Invalid value\n");
+
+	snprintf(buf, sizeof(buf), "write%c 0x%lx 0x%x\n", type, addr, val);
+	/* write to qtest socket */
+	ret = qtest_raw_send(s->qtest_socket, buf, strlen(buf));
+	/* read reply from event handler */
+	ret = qtest_raw_recv(s->msgfds.readfd, buf, sizeof(buf));
+}
+
+/*
+ * qtest_in/out are used for accessing ioport of qemu guest.
+ * qtest_read/write are used for accessing memory of qemu guest.
+ */
+uint32_t
+qtest_in(struct qtest_session *s, uint16_t addr, char type)
+{
+	uint32_t val;
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	val = qtest_raw_in(s, addr, type);
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	return val;
+}
+
+void
+qtest_out(struct qtest_session *s, uint16_t addr, uint64_t val, char type)
+{
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	qtest_raw_out(s, addr, val, type);
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+}
+
+uint32_t
+qtest_read(struct qtest_session *s, uint64_t addr, char type)
+{
+	uint32_t val;
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	val = qtest_raw_read(s, addr, type);
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	return val;
+}
+
+void
+qtest_write(struct qtest_session *s, uint64_t addr, uint64_t val, char type)
+{
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	qtest_raw_write(s, addr, val, type);
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+}
+
+static void
+qtest_event_send(struct qtest_session *s, char *buf)
+{
+	int ret;
+
+	/* relay normal message to pipe */
+	ret = qtest_raw_send(s->msgfds.writefd, buf, strlen(buf));
+	if (ret < 0)
+		rte_panic("cannot relay normal message\n");
+}
+
+static void
+qtest_close_one_socket(int *fd)
+{
+	if (*fd > 0) {
+		close(*fd);
+		*fd = -1;
+	}
+}
+
+static void
+qtest_close_sockets(struct qtest_session *s)
+{
+	qtest_close_one_socket(&s->qtest_socket);
+	qtest_close_one_socket(&s->msgfds.readfd);
+	qtest_close_one_socket(&s->msgfds.writefd);
+}
+
+static void
+qtest_event_enqueue(struct qtest_session *s, char *buf)
+{
+	size_t len = strlen(buf);
+	char *dest;
+
+	if (s->evq == NULL) {
+		/* allocate one more byte for '\0' */
+		s->evq = malloc(len + 1);
+		if (s->evq == NULL)
+			rte_panic("Cannot allocate memory\n");
+
+		s->evq_dequeue_ptr = s->evq;
+		s->evq_total_len = len + 1;
+		dest = s->evq;
+	} else {
+		size_t offset = s->evq_dequeue_ptr - s->evq;
+
+		s->evq = realloc(s->evq, s->evq_total_len + len);
+		if (s->evq == NULL)
+			rte_panic("Cannot re-allocate memory\n");
+
+		s->evq_dequeue_ptr = s->evq + offset;
+		dest = s->evq + s->evq_total_len - 1;
+		s->evq_total_len += len;
+	}
+
+	strncpy(dest, buf, len);
+	dest[len] = '\0';
+}
+
+static char *
+qtest_event_dequeue(struct qtest_session *s)
+{
+	char *head, *next_head;
+
+	head = s->evq_dequeue_ptr;
+
+	/* make sure message is terminated by '\n' */
+	next_head = strchr(s->evq_dequeue_ptr, '\n');
+	if (next_head == NULL)
+		return NULL;
+
+	/* set next dequeue pointer */
+	s->evq_dequeue_ptr = next_head + 1;
+
+	return head;
+}
+
+static void
+qtest_event_flush(struct qtest_session *s)
+{
+	if (s->evq) {
+		free(s->evq);
+		s->evq = NULL;
+		s->evq_dequeue_ptr = NULL;
+		s->evq_total_len = 0;
+	}
+}
+
+/*
+ * This thread relays QTest response using pipe and eventfd.
+ * The function is needed because we need to separate IRQ message from others.
+ */
+static void *
+qtest_event_handler(void *data) {
+	struct qtest_session *s = (struct qtest_session *)data;
+	char buf[64];
+	char *p;
+	int ret;
+
+	for (;;) {
+		memset(buf, 0, sizeof(buf));
+		ret = qtest_raw_recv(s->qtest_socket, buf, sizeof(buf));
+		if (ret <= 0) {
+			PMD_DRV_LOG(EMERG,
+				"QTest connection was closed.\n"
+				"Please detach the port, then start QEMU "
+				"and attach the port again.\n");
+			qtest_close_sockets(s);
+			qtest_event_flush(s);
+			return NULL;
+		}
+
+		qtest_event_enqueue(s, buf);
+
+		/* in the case of incomplete message, receive again */
+		p = &buf[sizeof(buf) - 1];
+		if ((*p != '\0') && (*p != '\n'))
+			continue;
+
+		/* may receive multiple messages at the same time */
+		while ((p = qtest_event_dequeue(s)) != NULL)
+			qtest_event_send(s, p);
+
+		qtest_event_flush(s);
+	}
+	return NULL;
+}
+
+static int
+qtest_open_socket(char *path)
+{
+	struct sockaddr_un sa = {0};
+	int ret, fd, loop = 100;
+
+	fd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (fd < 0)
+		return -1;
+
+	sa.sun_family = AF_UNIX;
+	strncpy(sa.sun_path, path, sizeof(sa.sun_path));
+
+	while (loop--) {
+		/*
+		 * If QEMU has multiple sockets needed to be listen, QEMU needs
+		 * some time to start listening a next socket.
+		 * In our case, after connecting ivshmem socket, we may need to wait
+		 * a bit to connect to qtest socket.
+		 */
+		ret = connect(fd, (struct sockaddr *)&sa,
+				sizeof(struct sockaddr_un));
+		if (ret == 0)
+			break;
+		else
+			usleep(100000);
+	}
+
+	if (ret != 0) {
+		close(fd);
+		return -1;
+	}
+
+	return fd;
+}
+
+void
+qtest_vdev_uninit(struct qtest_session *s)
+{
+	qtest_close_sockets(s);
+	qtest_event_flush(s);
+
+	if (s->event_th_started) {
+		pthread_cancel(s->event_th);
+		pthread_join(s->event_th, NULL);
+		s->event_th_started = 0;
+	}
+
+	pthread_mutex_destroy(&s->qtest_session_lock);
+	rte_free(s);
+}
+
+struct qtest_session *
+qtest_vdev_init(char *qtest_path)
+{
+	struct qtest_session *s;
+	int ret;
+
+	s = rte_zmalloc(NULL, sizeof(*s), RTE_CACHE_LINE_SIZE);
+	if (s == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to allocate memory\n");
+		return NULL;
+	}
+
+	ret = pthread_mutex_init(&s->qtest_session_lock, NULL);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to initialize mutex\n");
+		rte_free(s);
+		return NULL;
+	}
+
+	ret = pipe(s->msgfds.pipefd);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to initialize message pipe\n");
+		goto error;
+	}
+
+	s->qtest_socket = qtest_open_socket(qtest_path);
+	if (s->qtest_socket < 0) {
+		PMD_DRV_LOG(ERR, "Failed to open %s\n", qtest_path);
+		goto error;
+	}
+
+	ret = pthread_create(&s->event_th, NULL, qtest_event_handler, s);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to create event handler\n");
+		goto error;
+	}
+	s->event_th_started = 1;
+
+	return s;
+
+error:
+	qtest_vdev_uninit(s);
+	return NULL;
+}
diff --git a/drivers/net/virtio/qtest_utils.h b/drivers/net/virtio/qtest_utils.h
new file mode 100644
index 0000000..962fc5c
--- /dev/null
+++ b/drivers/net/virtio/qtest_utils.h
@@ -0,0 +1,119 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 IGEL Co., Ltd. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of IGEL Co., Ltd. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _VIRTIO_QTEST_UTILS_H_
+#define _VIRTIO_QTEST_UTILS_H_
+
+/**
+ * @internal
+ * Initialization function of QTest utility.
+ *
+ * @param qtest_path
+ *   Path of qtest socket.
+ * @return
+ *   The pointer to qtest session structure.
+ */
+struct qtest_session *qtest_vdev_init(char *qtest_path);
+
+/**
+ * @internal
+ * Finalization function of QTest utility.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ */
+void qtest_vdev_uninit(struct qtest_session *s);
+
+/**
+ * @internal
+ * Read a port of QEMU guest.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @param addr
+ *   The port address.
+ * @param type
+ *   Size of port. Specify one of 'l', 'w', and 'b'.
+ * @return
+ *   Value read from the port.
+ */
+uint32_t qtest_in(struct qtest_session *s, uint16_t addr, char type);
+
+/**
+ * @internal
+ * Write a port of QEMU guest.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @param addr
+ *   The port address.
+ * @param val
+ *   Written value.
+ * @param type
+ *   Size of port. Specify one of 'l', 'w', and 'b'.
+ */
+void qtest_out(struct qtest_session *s, uint16_t addr,
+			uint64_t val, char type);
+
+/**
+ * @internal
+ * Read memory of QEMU guest.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @param addr
+ *   The memory address.
+ * @param type
+ *   Size of port. Specify one of 'l', 'w', and 'b'.
+ * @return
+ *   Value read from the memory.
+ */
+uint32_t qtest_read(struct qtest_session *s, uint64_t addr, char type);
+
+/**
+ * @internal
+ * Write memory of QEMU guest.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @param addr
+ *   The memory address.
+ * @param val
+ *   Written value.
+ * @param type
+ *   Size of memory. Specify one of 'l', 'w', and 'b'.
+ */
+void qtest_write(struct qtest_session *s, uint64_t addr,
+			uint64_t val, char type);
+
+#endif /* _VIRTIO_QTEST_UTILS_H_ */
-- 
2.1.4