[dpdk-dev] [PATCH v3 7/7] net/virtio_user: enable multiqueue with vhost kernel

[Jianfeng Tan]

With vhost kernel, to enable multiqueue, we need backend device
in kernel support multiqueue feature. Specifically, with tap
as the backend, as linux/Documentation/networking/tuntap.txt shows,
we check if tap supports IFF_MULTI_QUEUE feature.

And for vhost kernel, each queue pair has a vhost fd, and with a tap
fd binding this vhost fd. All tap fds are set with the same tap
interface name.

Signed-off-by: Jianfeng Tan <jianfeng.tan at intel.com>
---
 drivers/net/virtio/virtio_user/vhost_kernel.c    | 69 +++++++++++++++++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  1 +
 2 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/drivers/net/virtio/virtio_user/vhost_kernel.c b/drivers/net/virtio/virtio_user/vhost_kernel.c
index bdb4af2..023bdf8 100644
--- a/drivers/net/virtio/virtio_user/vhost_kernel.c
+++ b/drivers/net/virtio/virtio_user/vhost_kernel.c
@@ -206,6 +206,29 @@ prepare_vhost_memory_kernel(void)
 	 (1ULL << VIRTIO_NET_F_CSUM))
 
 static int
+tap_supporte_mq(void)
+{
+	int tapfd;
+	unsigned int tap_features;
+
+	tapfd = open(PATH_NET_TUN, O_RDWR);
+	if (tapfd < 0) {
+		PMD_DRV_LOG(ERR, "fail to open %s: %s",
+			    PATH_NET_TUN, strerror(errno));
+		return -1;
+	}
+
+	if (ioctl(tapfd, TUNGETFEATURES, &tap_features) == -1) {
+		PMD_DRV_LOG(ERR, "TUNGETFEATURES failed: %s", strerror(errno));
+		close(tapfd);
+		return -1;
+	}
+
+	close(tapfd);
+	return tap_features & IFF_MULTI_QUEUE;
+}
+
+static int
 vhost_kernel_ioctl(struct virtio_user_dev *dev,
 		   enum vhost_user_request req,
 		   void *arg)
@@ -213,6 +236,8 @@ vhost_kernel_ioctl(struct virtio_user_dev *dev,
 	int i, ret = -1;
 	uint64_t req_kernel;
 	struct vhost_memory_kernel *vm = NULL;
+	int vhostfd;
+	unsigned int queue_sel;
 
 	PMD_DRV_LOG(INFO, "%s", vhost_msg_strings[req]);
 
@@ -232,15 +257,37 @@ vhost_kernel_ioctl(struct virtio_user_dev *dev,
 		/* VHOST kernel does not know about below flags */
 		*(uint64_t *)arg &= ~VHOST_KERNEL_GUEST_OFFLOADS_MASK;
 		*(uint64_t *)arg &= ~VHOST_KERNEL_HOST_OFFLOADS_MASK;
+
+		*(uint64_t *)arg &= ~(1ULL << VIRTIO_NET_F_MQ);
 	}
 
-	for (i = 0; i < VHOST_KERNEL_MAX_QUEUES; ++i) {
-		if (dev->vhostfds[i] < 0)
-			continue;
+	switch (req_kernel) {
+	case VHOST_SET_VRING_NUM:
+	case VHOST_SET_VRING_ADDR:
+	case VHOST_SET_VRING_BASE:
+	case VHOST_GET_VRING_BASE:
+	case VHOST_SET_VRING_KICK:
+	case VHOST_SET_VRING_CALL:
+		queue_sel = *(unsigned int *)arg;
+		vhostfd = dev->vhostfds[queue_sel / 2];
+		*(unsigned int *)arg = queue_sel % 2;
+		PMD_DRV_LOG(DEBUG, "vhostfd=%d, index=%u",
+			    vhostfd, *(unsigned int *)arg);
+		break;
+	default:
+		vhostfd = -1;
+	}
+	if (vhostfd == -1) {
+		for (i = 0; i < VHOST_KERNEL_MAX_QUEUES; ++i) {
+			if (dev->vhostfds[i] < 0)
+				continue;
 
-		ret = ioctl(dev->vhostfds[i], req_kernel, arg);
-		if (ret < 0)
-			break;
+			ret = ioctl(dev->vhostfds[i], req_kernel, arg);
+			if (ret < 0)
+				break;
+		}
+	} else {
+		ret = ioctl(vhostfd, req_kernel, arg);
 	}
 
 	if (!ret && req_kernel == VHOST_GET_FEATURES) {
@@ -250,6 +297,12 @@ vhost_kernel_ioctl(struct virtio_user_dev *dev,
 		 */
 		*((uint64_t *)arg) |= VHOST_KERNEL_GUEST_OFFLOADS_MASK;
 		*((uint64_t *)arg) |= VHOST_KERNEL_HOST_OFFLOADS_MASK;
+
+		/* vhost_kernel will not declare this feature, but it does
+		 * support multi-queue.
+		 */
+		if (tap_supporte_mq())
+			*(uint64_t *)arg |= (1ull << VIRTIO_NET_F_MQ);
 	}
 
 	if (vm)
@@ -329,6 +382,7 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 			TUN_F_TSO6 |
 			TUN_F_TSO_ECN |
 			TUN_F_UFO;
+	int req_mq = (dev->max_queue_pairs > 1);
 
 	vhostfd = dev->vhostfds[pair_idx];
 
@@ -382,6 +436,9 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 		goto error;
 	}
 
+	if (req_mq)
+		ifr.ifr_flags |= IFF_MULTI_QUEUE;
+
 	if (dev->ifname)
 		strncpy(ifr.ifr_name, dev->ifname, IFNAMSIZ);
 	else
diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c
index c40b77e..2d9d989 100644
--- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
+++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
@@ -93,6 +93,7 @@ virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel)
 	state.num = vring->num;
 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state);
 
+	state.index = queue_sel;
 	state.num = 0; /* no reservation */
 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state);
 
-- 
2.7.4