[dpdk-dev] [PATCH 06/28] vhost: move vhost-user connection

Nikos Dragazis ndragazis at arrikto.com
Wed Jun 19 17:14:31 CEST 2019


The AF_UNIX transport can accept multiple client connections on a server
socket.  Each connection instantiates a separate vhost-user device,
which is stored as a vhost_user_connection.  This behavior is specific
to AF_UNIX and other transports may not support N connections per
socket endpoint.

Move struct vhost_user_connection to trans_af_unix.c and
conn_list/conn_mutex into struct af_unix_socket.

Signed-off-by: Nikos Dragazis <ndragazis at arrikto.com>
Signed-off-by: Stefan Hajnoczi <stefanha at redhat.com>
---
 lib/librte_vhost/socket.c        | 54 +++---------------------------
 lib/librte_vhost/trans_af_unix.c | 72 ++++++++++++++++++++++++++++++++++++----
 lib/librte_vhost/vhost.h         | 19 ++---------
 3 files changed, 74 insertions(+), 71 deletions(-)

diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index df6d707..976343c 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -341,13 +341,6 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 		vhost_user_socket_mem_free(vsocket);
 		goto out;
 	}
-	TAILQ_INIT(&vsocket->conn_list);
-	ret = pthread_mutex_init(&vsocket->conn_mutex, NULL);
-	if (ret) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"error: failed to init connection mutex\n");
-		goto out_free;
-	}
 	vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
 
 	/*
@@ -395,7 +388,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 		RTE_LOG(ERR, VHOST_CONFIG,
 			"Postcopy requested but not compiled\n");
 		ret = -1;
-		goto out_mutex;
+		goto out_free;
 #endif
 	}
 
@@ -403,14 +396,14 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 		vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
 		if (vsocket->reconnect && reconn_tid == 0) {
 			if (vhost_user_reconnect_init() != 0)
-				goto out_mutex;
+				goto out_free;
 		}
 	} else {
 		vsocket->is_server = true;
 	}
 	ret = trans_ops->socket_init(vsocket, flags);
 	if (ret < 0) {
-		goto out_mutex;
+		goto out_free;
 	}
 
 	vhost_user.vsockets[vhost_user.vsocket_cnt++] = vsocket;
@@ -418,11 +411,6 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 	pthread_mutex_unlock(&vhost_user.mutex);
 	return ret;
 
-out_mutex:
-	if (pthread_mutex_destroy(&vsocket->conn_mutex)) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"error: failed to destroy connection mutex\n");
-	}
 out_free:
 	vhost_user_socket_mem_free(vsocket);
 out:
@@ -439,51 +427,19 @@ rte_vhost_driver_unregister(const char *path)
 {
 	int i;
 	int count;
-	struct vhost_user_connection *conn, *next;
 
 	if (path == NULL)
 		return -1;
 
-again:
 	pthread_mutex_lock(&vhost_user.mutex);
 
 	for (i = 0; i < vhost_user.vsocket_cnt; i++) {
 		struct vhost_user_socket *vsocket = vhost_user.vsockets[i];
 
 		if (!strcmp(vsocket->path, path)) {
-			pthread_mutex_lock(&vsocket->conn_mutex);
-			for (conn = TAILQ_FIRST(&vsocket->conn_list);
-			     conn != NULL;
-			     conn = next) {
-				next = TAILQ_NEXT(conn, next);
-
-				/*
-				 * If r/wcb is executing, release the
-				 * conn_mutex lock, and try again since
-				 * the r/wcb may use the conn_mutex lock.
-				 */
-				if (fdset_try_del(&vhost_user.fdset,
-						  conn->connfd) == -1) {
-					pthread_mutex_unlock(
-							&vsocket->conn_mutex);
-					pthread_mutex_unlock(&vhost_user.mutex);
-					goto again;
-				}
-
-				RTE_LOG(INFO, VHOST_CONFIG,
-					"free connfd = %d for device '%s'\n",
-					conn->connfd, path);
-				close(conn->connfd);
-				vhost_destroy_device(conn->vid);
-				TAILQ_REMOVE(&vsocket->conn_list, conn, next);
-				free(conn);
-			}
-			pthread_mutex_unlock(&vsocket->conn_mutex);
-
 			vsocket->trans_ops->socket_cleanup(vsocket);
-
-			pthread_mutex_destroy(&vsocket->conn_mutex);
-			vhost_user_socket_mem_free(vsocket);
+			free(vsocket->path);
+			free(vsocket);
 
 			count = --vhost_user.vsocket_cnt;
 			vhost_user.vsockets[i] = vhost_user.vsockets[count];
diff --git a/lib/librte_vhost/trans_af_unix.c b/lib/librte_vhost/trans_af_unix.c
index 93d11f7..58fc9e2 100644
--- a/lib/librte_vhost/trans_af_unix.c
+++ b/lib/librte_vhost/trans_af_unix.c
@@ -15,8 +15,20 @@
 
 #define MAX_VIRTIO_BACKLOG 128
 
+TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
+
+struct vhost_user_connection {
+	struct vhost_user_socket *vsocket;
+	int connfd;
+	int vid;
+
+	TAILQ_ENTRY(vhost_user_connection) next;
+};
+
 struct af_unix_socket {
 	struct vhost_user_socket socket; /* must be the first field! */
+	struct vhost_user_connection_list conn_list;
+	pthread_mutex_t conn_mutex;
 	int socket_fd;
 	struct sockaddr_un un;
 };
@@ -131,6 +143,8 @@ send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num)
 static void
 vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
 {
+	struct af_unix_socket *af_vsocket =
+		container_of(vsocket, struct af_unix_socket, socket);
 	int vid;
 	size_t size;
 	struct vhost_user_connection *conn;
@@ -188,9 +202,9 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
 		goto err_cleanup;
 	}
 
-	pthread_mutex_lock(&vsocket->conn_mutex);
-	TAILQ_INSERT_TAIL(&vsocket->conn_list, conn, next);
-	pthread_mutex_unlock(&vsocket->conn_mutex);
+	pthread_mutex_lock(&af_vsocket->conn_mutex);
+	TAILQ_INSERT_TAIL(&af_vsocket->conn_list, conn, next);
+	pthread_mutex_unlock(&af_vsocket->conn_mutex);
 
 	fdset_pipe_notify(&vhost_user.fdset);
 	return;
@@ -221,6 +235,8 @@ vhost_user_read_cb(int connfd, void *dat, int *remove)
 {
 	struct vhost_user_connection *conn = dat;
 	struct vhost_user_socket *vsocket = conn->vsocket;
+	struct af_unix_socket *af_vsocket =
+		container_of(vsocket, struct af_unix_socket, socket);
 	int ret;
 
 	ret = vhost_user_msg_handler(conn->vid, connfd);
@@ -238,9 +254,9 @@ vhost_user_read_cb(int connfd, void *dat, int *remove)
 
 		vhost_destroy_device(conn->vid);
 
-		pthread_mutex_lock(&vsocket->conn_mutex);
-		TAILQ_REMOVE(&vsocket->conn_list, conn, next);
-		pthread_mutex_unlock(&vsocket->conn_mutex);
+		pthread_mutex_lock(&af_vsocket->conn_mutex);
+		TAILQ_REMOVE(&af_vsocket->conn_list, conn, next);
+		pthread_mutex_unlock(&af_vsocket->conn_mutex);
 
 		free(conn);
 
@@ -512,6 +528,18 @@ static int
 af_unix_socket_init(struct vhost_user_socket *vsocket,
 		    uint64_t flags __rte_unused)
 {
+	struct af_unix_socket *af_vsocket =
+		container_of(vsocket, struct af_unix_socket, socket);
+	int ret;
+
+	TAILQ_INIT(&af_vsocket->conn_list);
+	ret = pthread_mutex_init(&af_vsocket->conn_mutex, NULL);
+	if (ret) {
+		RTE_LOG(ERR, VHOST_CONFIG,
+			"error: failed to init connection mutex\n");
+		return -1;
+	}
+
 	return create_unix_socket(vsocket);
 }
 
@@ -520,6 +548,7 @@ af_unix_socket_cleanup(struct vhost_user_socket *vsocket)
 {
 	struct af_unix_socket *af_vsocket =
 		container_of(vsocket, struct af_unix_socket, socket);
+	struct vhost_user_connection *conn, *next;
 
 	if (vsocket->is_server) {
 		fdset_del(&vhost_user.fdset, af_vsocket->socket_fd);
@@ -528,6 +557,37 @@ af_unix_socket_cleanup(struct vhost_user_socket *vsocket)
 	} else if (vsocket->reconnect) {
 		vhost_user_remove_reconnect(vsocket);
 	}
+
+again:
+	pthread_mutex_lock(&af_vsocket->conn_mutex);
+	for (conn = TAILQ_FIRST(&af_vsocket->conn_list);
+	     conn != NULL;
+	     conn = next) {
+		next = TAILQ_NEXT(conn, next);
+
+		/*
+		 * If r/wcb is executing, release the
+		 * conn_mutex lock, and try again since
+		 * the r/wcb may use the conn_mutex lock.
+		 */
+		if (fdset_try_del(&vhost_user.fdset,
+				  conn->connfd) == -1) {
+			pthread_mutex_unlock(
+					&af_vsocket->conn_mutex);
+			goto again;
+		}
+
+		RTE_LOG(INFO, VHOST_CONFIG,
+			"free connfd = %d for device '%s'\n",
+			conn->connfd, vsocket->path);
+		close(conn->connfd);
+		vhost_destroy_device(conn->vid);
+		TAILQ_REMOVE(&af_vsocket->conn_list, conn, next);
+		free(conn);
+	}
+	pthread_mutex_unlock(&af_vsocket->conn_mutex);
+
+	pthread_mutex_destroy(&af_vsocket->conn_mutex);
 }
 
 static int
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index c74753b..5c3987d 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -404,13 +404,10 @@ struct virtio_net {
 	struct rte_vhost_user_extern_ops extern_ops;
 } __rte_cache_aligned;
 
-/* The vhost_user, vhost_user_socket, vhost_user_connection, and reconnect
- * declarations are temporary measures for moving AF_UNIX code into
- * trans_af_unix.c.  They will be cleaned up as socket.c is untangled from
- * trans_af_unix.c.
+/* The vhost_user, vhost_user_socket, and reconnect declarations are temporary
+ * measures for moving AF_UNIX code into trans_af_unix.c.  They will be cleaned
+ * up as socket.c is untangled from trans_af_unix.c.
  */
-TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
-
 /*
  * Every time rte_vhost_driver_register() is invoked, an associated
  * vhost_user_socket struct will be created.
@@ -421,8 +418,6 @@ TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
  * struct.
  */
 struct vhost_user_socket {
-	struct vhost_user_connection_list conn_list;
-	pthread_mutex_t conn_mutex;
 	char *path;
 	bool is_server;
 	bool reconnect;
@@ -453,14 +448,6 @@ struct vhost_user_socket {
 	struct vhost_transport_ops const *trans_ops;
 };
 
-struct vhost_user_connection {
-	struct vhost_user_socket *vsocket;
-	int connfd;
-	int vid;
-
-	TAILQ_ENTRY(vhost_user_connection) next;
-};
-
 #define MAX_VHOST_SOCKET 1024
 struct vhost_user {
 	struct vhost_user_socket *vsockets[MAX_VHOST_SOCKET];
-- 
2.7.4



More information about the dev mailing list