From patchwork Wed Jun 19 15:14:36 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Nikos Dragazis X-Patchwork-Id: 54965 X-Patchwork-Delegate: maxime.coquelin@redhat.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 233671C3E8; Wed, 19 Jun 2019 17:16:10 +0200 (CEST) Received: from mx0.arrikto.com (mx0.arrikto.com [212.71.252.59]) by dpdk.org (Postfix) with ESMTP id A5DA91C389 for ; Wed, 19 Jun 2019 17:15:40 +0200 (CEST) Received: from troi.prod.arr (mail.arr [10.99.0.5]) by mx0.arrikto.com (Postfix) with ESMTP id 65685182010; Wed, 19 Jun 2019 18:15:40 +0300 (EEST) Received: from localhost.localdomain (unknown [10.89.50.133]) by troi.prod.arr (Postfix) with ESMTPSA id A57D632C; Wed, 19 Jun 2019 18:15:39 +0300 (EEST) From: Nikos Dragazis To: dev@dpdk.org Cc: Maxime Coquelin , Tiwei Bie , Zhihong Wang , Stefan Hajnoczi , Wei Wang , Stojaczyk Dariusz , Vangelis Koukis Date: Wed, 19 Jun 2019 18:14:36 +0300 Message-Id: <1560957293-17294-12-git-send-email-ndragazis@arrikto.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1560957293-17294-1-git-send-email-ndragazis@arrikto.com> References: <1560957293-17294-1-git-send-email-ndragazis@arrikto.com> Subject: [dpdk-dev] [PATCH 11/28] vhost: extract socket I/O into transport X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" The core vhost-user protocol code should not do socket I/O, because the details are transport-specific. Move code to send and receive vhost-user messages into trans_af_unix.c. The connection fd is a transport-specific feature. Therefore, it should and eventually will be removed from the core vhost-user code. That is, it will be removed from the vhost_user_msg_handler() and the message handlers. We keep it for now, because vhost_user_set_mem_table() needs it. In a later commit, we will refactor the map/unmap functionality and after that we will be able to remove the connection fds from the core vhost-user code. Signed-off-by: Nikos Dragazis Signed-off-by: Stefan Hajnoczi --- lib/librte_vhost/trans_af_unix.c | 70 +++++++++++++++++++++++++++++++++--- lib/librte_vhost/vhost.h | 26 ++++++++++++++ lib/librte_vhost/vhost_user.c | 78 ++++++++-------------------------------- lib/librte_vhost/vhost_user.h | 7 +--- 4 files changed, 108 insertions(+), 73 deletions(-) diff --git a/lib/librte_vhost/trans_af_unix.c b/lib/librte_vhost/trans_af_unix.c index 7e119b4..c0ba8df 100644 --- a/lib/librte_vhost/trans_af_unix.c +++ b/lib/librte_vhost/trans_af_unix.c @@ -50,7 +50,7 @@ static void vhost_user_read_cb(int connfd, void *dat, int *remove); * return bytes# of read on success or negative val on failure. Update fdnum * with number of fds read. */ -int +static int read_fd_message(int sockfd, char *buf, int buflen, int *fds, int max_fds, int *fd_num) { @@ -101,8 +101,8 @@ read_fd_message(int sockfd, char *buf, int buflen, int *fds, int max_fds, return ret; } -int -send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num) +static int +send_fd_message(int sockfd, void *buf, int buflen, int *fds, int fd_num) { struct iovec iov; struct msghdr msgh; @@ -148,6 +148,23 @@ send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num) return ret; } +static int +af_unix_send_reply(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + struct vhost_user_connection *conn = + container_of(dev, struct vhost_user_connection, device); + + return send_fd_message(conn->connfd, msg, + VHOST_USER_HDR_SIZE + msg->size, msg->fds, msg->fd_num); +} + +static int +af_unix_send_slave_req(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + return send_fd_message(dev->slave_req_fd, msg, + VHOST_USER_HDR_SIZE + msg->size, msg->fds, msg->fd_num); +} + static void vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) { @@ -231,6 +248,36 @@ vhost_user_server_new_connection(int fd, void *dat, int *remove __rte_unused) vhost_user_add_connection(fd, vsocket); } +/* return bytes# of read on success or negative val on failure. */ +int +read_vhost_message(int sockfd, struct VhostUserMsg *msg) +{ + int ret; + + ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE, + msg->fds, VHOST_MEMORY_MAX_NREGIONS, &msg->fd_num); + if (ret <= 0) + return ret; + + if (msg->size) { + if (msg->size > sizeof(msg->payload)) { + RTE_LOG(ERR, VHOST_CONFIG, + "invalid msg size: %d\n", msg->size); + return -1; + } + ret = read(sockfd, &msg->payload, msg->size); + if (ret <= 0) + return ret; + if (ret != (int)msg->size) { + RTE_LOG(ERR, VHOST_CONFIG, + "read control message failed\n"); + return -1; + } + } + + return ret; +} + static void vhost_user_read_cb(int connfd, void *dat, int *remove) { @@ -238,10 +285,23 @@ vhost_user_read_cb(int connfd, void *dat, int *remove) struct vhost_user_socket *vsocket = conn->vsocket; struct af_unix_socket *af_vsocket = container_of(vsocket, struct af_unix_socket, socket); + struct VhostUserMsg msg; int ret; - ret = vhost_user_msg_handler(conn->device.vid, connfd); + ret = read_vhost_message(connfd, &msg); + if (ret <= 0) { + if (ret < 0) + RTE_LOG(ERR, VHOST_CONFIG, + "vhost read message failed\n"); + else if (ret == 0) + RTE_LOG(INFO, VHOST_CONFIG, + "vhost peer closed\n"); + goto err; + } + + ret = vhost_user_msg_handler(conn->device.vid, connfd, &msg); if (ret < 0) { +err: close(connfd); *remove = 1; @@ -638,4 +698,6 @@ const struct vhost_transport_ops af_unix_trans_ops = { .socket_cleanup = af_unix_socket_cleanup, .socket_start = af_unix_socket_start, .vring_call = af_unix_vring_call, + .send_reply = af_unix_send_reply, + .send_slave_req = af_unix_send_slave_req, }; diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index b9e4df1..b20773c 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -290,6 +290,7 @@ struct guest_page { struct virtio_net; struct vhost_user_socket; +struct VhostUserMsg; /** * A structure containing function pointers for transport-specific operations. @@ -351,6 +352,31 @@ struct vhost_transport_ops { * 0 on success, -1 on failure */ int (*vring_call)(struct virtio_net *dev, struct vhost_virtqueue *vq); + + /** + * Send a reply to the master. + * + * @param dev + * vhost device + * @param reply + * reply message + * @return + * 0 on success, -1 on failure + */ + int (*send_reply)(struct virtio_net *dev, struct VhostUserMsg *reply); + + /** + * Send a slave request to the master. + * + * @param dev + * vhost device + * @param req + * request message + * @return + * 0 on success, -1 on failure + */ + int (*send_slave_req)(struct virtio_net *dev, + struct VhostUserMsg *req); }; /** The traditional AF_UNIX vhost-user protocol transport. */ diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c index c9e29ec..5c12435 100644 --- a/lib/librte_vhost/vhost_user.c +++ b/lib/librte_vhost/vhost_user.c @@ -80,8 +80,8 @@ static const char *vhost_message_str[VHOST_USER_MAX] = { [VHOST_USER_POSTCOPY_END] = "VHOST_USER_POSTCOPY_END", }; -static int send_vhost_reply(int sockfd, struct VhostUserMsg *msg); -static int read_vhost_message(int sockfd, struct VhostUserMsg *msg); +static int send_vhost_reply(struct virtio_net *dev, struct VhostUserMsg *msg); +int read_vhost_message(int sockfd, struct VhostUserMsg *msg); static uint64_t get_blk_size(int fd) @@ -1042,7 +1042,7 @@ vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *msg, if (dev->postcopy_listening) { /* Send the addresses back to qemu */ msg->fd_num = 0; - send_vhost_reply(main_fd, msg); + send_vhost_reply(dev, msg); /* Wait for qemu to acknolwedge it's got the addresses * we've got to wait before we're allowed to generate faults. @@ -1764,49 +1764,8 @@ static vhost_message_handler_t vhost_message_handlers[VHOST_USER_MAX] = { [VHOST_USER_POSTCOPY_END] = vhost_user_postcopy_end, }; - -/* return bytes# of read on success or negative val on failure. */ static int -read_vhost_message(int sockfd, struct VhostUserMsg *msg) -{ - int ret; - - ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE, - msg->fds, VHOST_MEMORY_MAX_NREGIONS, &msg->fd_num); - if (ret <= 0) - return ret; - - if (msg->size) { - if (msg->size > sizeof(msg->payload)) { - RTE_LOG(ERR, VHOST_CONFIG, - "invalid msg size: %d\n", msg->size); - return -1; - } - ret = read(sockfd, &msg->payload, msg->size); - if (ret <= 0) - return ret; - if (ret != (int)msg->size) { - RTE_LOG(ERR, VHOST_CONFIG, - "read control message failed\n"); - return -1; - } - } - - return ret; -} - -static int -send_vhost_message(int sockfd, struct VhostUserMsg *msg) -{ - if (!msg) - return 0; - - return send_fd_message(sockfd, (char *)msg, - VHOST_USER_HDR_SIZE + msg->size, msg->fds, msg->fd_num); -} - -static int -send_vhost_reply(int sockfd, struct VhostUserMsg *msg) +send_vhost_reply(struct virtio_net *dev, struct VhostUserMsg *msg) { if (!msg) return 0; @@ -1816,7 +1775,7 @@ send_vhost_reply(int sockfd, struct VhostUserMsg *msg) msg->flags |= VHOST_USER_VERSION; msg->flags |= VHOST_USER_REPLY_MASK; - return send_vhost_message(sockfd, msg); + return dev->trans_ops->send_reply(dev, msg); } static int @@ -1827,7 +1786,7 @@ send_vhost_slave_message(struct virtio_net *dev, struct VhostUserMsg *msg) if (msg->flags & VHOST_USER_NEED_REPLY) rte_spinlock_lock(&dev->slave_req_lock); - ret = send_vhost_message(dev->slave_req_fd, msg); + ret = dev->trans_ops->send_slave_req(dev, msg); if (ret < 0 && (msg->flags & VHOST_USER_NEED_REPLY)) rte_spinlock_unlock(&dev->slave_req_lock); @@ -1908,10 +1867,10 @@ vhost_user_unlock_all_queue_pairs(struct virtio_net *dev) } int -vhost_user_msg_handler(int vid, int fd) +vhost_user_msg_handler(int vid, int fd, const struct VhostUserMsg *msg_) { + struct VhostUserMsg msg = *msg_; /* copy so we can build the reply */ struct virtio_net *dev; - struct VhostUserMsg msg; struct rte_vdpa_device *vdpa_dev; int did = -1; int ret; @@ -1933,15 +1892,8 @@ vhost_user_msg_handler(int vid, int fd) } } - ret = read_vhost_message(fd, &msg); - if (ret <= 0) { - if (ret < 0) - RTE_LOG(ERR, VHOST_CONFIG, - "vhost read message failed\n"); - else - RTE_LOG(INFO, VHOST_CONFIG, - "vhost peer closed\n"); - + if (msg.request.master >= VHOST_USER_MAX) { + RTE_LOG(ERR, VHOST_CONFIG, "vhost read incorrect message\n"); return -1; } @@ -2004,7 +1956,7 @@ vhost_user_msg_handler(int vid, int fd) (void *)&msg); switch (ret) { case RTE_VHOST_MSG_RESULT_REPLY: - send_vhost_reply(fd, &msg); + send_vhost_reply(dev, &msg); /* Fall-through */ case RTE_VHOST_MSG_RESULT_ERR: case RTE_VHOST_MSG_RESULT_OK: @@ -2038,7 +1990,7 @@ vhost_user_msg_handler(int vid, int fd) RTE_LOG(DEBUG, VHOST_CONFIG, "Processing %s succeeded and needs reply.\n", vhost_message_str[request]); - send_vhost_reply(fd, &msg); + send_vhost_reply(dev, &msg); handled = true; break; default: @@ -2053,7 +2005,7 @@ vhost_user_msg_handler(int vid, int fd) (void *)&msg); switch (ret) { case RTE_VHOST_MSG_RESULT_REPLY: - send_vhost_reply(fd, &msg); + send_vhost_reply(dev, &msg); /* Fall-through */ case RTE_VHOST_MSG_RESULT_ERR: case RTE_VHOST_MSG_RESULT_OK: @@ -2083,7 +2035,7 @@ vhost_user_msg_handler(int vid, int fd) msg.payload.u64 = ret == RTE_VHOST_MSG_RESULT_ERR; msg.size = sizeof(msg.payload.u64); msg.fd_num = 0; - send_vhost_reply(fd, &msg); + send_vhost_reply(dev, &msg); } else if (ret == RTE_VHOST_MSG_RESULT_ERR) { RTE_LOG(ERR, VHOST_CONFIG, "vhost message handling failed.\n"); @@ -2161,7 +2113,7 @@ vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm) }, }; - ret = send_vhost_message(dev->slave_req_fd, &msg); + ret = send_vhost_slave_req(dev, &msg); if (ret < 0) { RTE_LOG(ERR, VHOST_CONFIG, "Failed to send IOTLB miss message (%d)\n", diff --git a/lib/librte_vhost/vhost_user.h b/lib/librte_vhost/vhost_user.h index 2a650fe..0169bd2 100644 --- a/lib/librte_vhost/vhost_user.h +++ b/lib/librte_vhost/vhost_user.h @@ -146,12 +146,7 @@ typedef struct VhostUserMsg { /* vhost_user.c */ -int vhost_user_msg_handler(int vid, int fd); +int vhost_user_msg_handler(int vid, int fd, const struct VhostUserMsg *msg); int vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm); -/* socket.c */ -int read_fd_message(int sockfd, char *buf, int buflen, int *fds, int max_fds, - int *fd_num); -int send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num); - #endif