[39/40] net/virtio: move Vhost-kernel data to its backend
Checks
Commit Message
As done earlier for Vhost-user, this patch moves the
Vhost-Kernel specific data to its backend file.
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
drivers/net/virtio/virtio_user/vhost_kernel.c | 78 +++++++++++++++----
.../net/virtio/virtio_user/virtio_user_dev.c | 43 ++--------
.../net/virtio/virtio_user/virtio_user_dev.h | 7 +-
3 files changed, 72 insertions(+), 56 deletions(-)
Comments
Hi Maxime,
> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Monday, December 21, 2020 5:14 AM
> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>; olivier.matz@6wind.com;
> amorenoz@redhat.com; david.marchand@redhat.com
> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
> Subject: [PATCH 39/40] net/virtio: move Vhost-kernel data to its backend
>
> As done earlier for Vhost-user, this patch moves the
> Vhost-Kernel specific data to its backend file.
>
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
> drivers/net/virtio/virtio_user/vhost_kernel.c | 78 +++++++++++++++----
> .../net/virtio/virtio_user/virtio_user_dev.c | 43 ++--------
> .../net/virtio/virtio_user/virtio_user_dev.h | 7 +-
> 3 files changed, 72 insertions(+), 56 deletions(-)
>
<snip>
> @@ -339,19 +353,37 @@ static int
> vhost_kernel_setup(struct virtio_user_dev *dev)
> {
> int vhostfd;
> - uint32_t i;
> + uint32_t q, i;
> + struct vhost_kernel_data *data;
> +
> + data = malloc(sizeof(*data));
> + if (!data) {
> + PMD_INIT_LOG(ERR, "(%s) Failed to allocate Vhost-kernel data",
> dev->path);
> + return -1;
> + }
> +
> + data->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
> + data->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
> + if (!data->vhostfds || !data->tapfds) {
> + PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
> + return -1;
> + }
> +
> + for (q = 0; q < dev->max_queue_pairs; ++q) {
> + data->vhostfds[q] = -1;
> + data->tapfds[q] = -1;
> + }
>
> get_vhost_kernel_max_regions();
>
> for (i = 0; i < dev->max_queue_pairs; ++i) {
> vhostfd = open(dev->path, O_RDWR);
> if (vhostfd < 0) {
> - PMD_DRV_LOG(ERR, "fail to open %s, %s",
> - dev->path, strerror(errno));
> + PMD_DRV_LOG(ERR, "fail to open %s, %s", dev->path,
> strerror(errno));
> return -1;
> }
In this function, should we clean-up malloced resources when error happens?
Thanks,
Chenbo
Hi Maxime,
> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Monday, December 21, 2020 5:14 AM
> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>; olivier.matz@6wind.com;
> amorenoz@redhat.com; david.marchand@redhat.com
> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
> Subject: [PATCH 39/40] net/virtio: move Vhost-kernel data to its backend
>
> As done earlier for Vhost-user, this patch moves the
> Vhost-Kernel specific data to its backend file.
>
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
> drivers/net/virtio/virtio_user/vhost_kernel.c | 78 +++++++++++++++----
> .../net/virtio/virtio_user/virtio_user_dev.c | 43 ++--------
> .../net/virtio/virtio_user/virtio_user_dev.h | 7 +-
> 3 files changed, 72 insertions(+), 56 deletions(-)
<snip>
> vhost_kernel_setup(struct virtio_user_dev *dev)
> {
> int vhostfd;
> - uint32_t i;
> + uint32_t q, i;
> + struct vhost_kernel_data *data;
> +
> + data = malloc(sizeof(*data));
> + if (!data) {
> + PMD_INIT_LOG(ERR, "(%s) Failed to allocate Vhost-kernel data",
> dev->path);
> + return -1;
> + }
> +
> + data->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
> + data->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
> + if (!data->vhostfds || !data->tapfds) {
> + PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
> + return -1;
> + }
> +
> + for (q = 0; q < dev->max_queue_pairs; ++q) {
> + data->vhostfds[q] = -1;
> + data->tapfds[q] = -1;
> + }
Forget to do 'dev->backend_data = data'? 😊
Besides,
When I do test 'virtio_user for exceptional path', iperf just cannot normally
send/recv pkts. Testpmd have some pkts (<10), guess tcp connection cannot be
established. I have not tried to locate the root cause. Just thinking it's
better to tell you first.
Thanks!
Chenbo
>
> get_vhost_kernel_max_regions();
>
> for (i = 0; i < dev->max_queue_pairs; ++i) {
> vhostfd = open(dev->path, O_RDWR);
> if (vhostfd < 0) {
> - PMD_DRV_LOG(ERR, "fail to open %s, %s",
> - dev->path, strerror(errno));
> + PMD_DRV_LOG(ERR, "fail to open %s, %s", dev->path,
> strerror(errno));
> return -1;
> }
>
> - dev->vhostfds[i] = vhostfd;
> + data->vhostfds[i] = vhostfd;
> }
>
> return 0;
> @@ -360,6 +392,19 @@ vhost_kernel_setup(struct virtio_user_dev *dev)
> static int
> vhost_kernel_destroy(struct virtio_user_dev *dev)
> {
> + struct vhost_kernel_data *data = dev->backend_data;
> + uint32_t i;
> +
> + for (i = 0; i < dev->max_queue_pairs; ++i) {
> + close(data->vhostfds[i]);
> + if (data->tapfds[i] >= 0)
> + close(data->tapfds[i]);
> + }
> +
> + free(data->vhostfds);
> + free(data->tapfds);
> + free(data);
> +
> return 0;
> }
>
> @@ -395,14 +440,15 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev
> *dev,
> int vhostfd;
> int tapfd;
> int req_mq = (dev->max_queue_pairs > 1);
> + struct vhost_kernel_data *data = dev->backend_data;
>
> - vhostfd = dev->vhostfds[pair_idx];
> + vhostfd = data->vhostfds[pair_idx];
>
> if (dev->qp_enabled[pair_idx] == enable)
> return 0;
>
> if (!enable) {
> - tapfd = dev->tapfds[pair_idx];
> + tapfd = data->tapfds[pair_idx];
> if (vhost_kernel_set_backend(vhostfd, -1) < 0) {
> PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
> return -1;
> @@ -415,8 +461,8 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
> return 0;
> }
>
> - if (dev->tapfds[pair_idx] >= 0) {
> - tapfd = dev->tapfds[pair_idx];
> + if (data->tapfds[pair_idx] >= 0) {
> + tapfd = data->tapfds[pair_idx];
> if (vhost_kernel_tap_set_offload(tapfd, dev->features) == -1)
> return -1;
> if (req_mq && vhost_kernel_tap_set_queue(tapfd, true) < 0) {
> @@ -439,7 +485,7 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
> return -1;
> }
>
> - dev->tapfds[pair_idx] = tapfd;
> + data->tapfds[pair_idx] = tapfd;
>
> set_backend:
> if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c
> b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> index 93a9ce2cd2..154aecc209 100644
> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> @@ -397,11 +397,6 @@ virtio_user_mem_event_cb(enum rte_mem_event type
> __rte_unused,
> static int
> virtio_user_dev_setup(struct virtio_user_dev *dev)
> {
> - uint32_t q;
> -
> - dev->vhostfds = NULL;
> - dev->tapfds = NULL;
> -
> if (dev->is_server) {
> if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) {
> PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!");
> @@ -409,34 +404,21 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
> }
> }
>
> - if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) {
> + switch (dev->backend_type) {
> + case VIRTIO_USER_BACKEND_VHOST_USER:
> dev->ops = &virtio_ops_user;
> - } else if (dev->backend_type ==
> - VIRTIO_USER_BACKEND_VHOST_KERNEL) {
> + break;
> + case VIRTIO_USER_BACKEND_VHOST_KERNEL:
> dev->ops = &virtio_ops_kernel;
> -
> - dev->vhostfds = malloc(dev->max_queue_pairs *
> - sizeof(int));
> - dev->tapfds = malloc(dev->max_queue_pairs *
> - sizeof(int));
> - if (!dev->vhostfds || !dev->tapfds) {
> - PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
> - return -1;
> - }
> -
> - for (q = 0; q < dev->max_queue_pairs; ++q) {
> - dev->vhostfds[q] = -1;
> - dev->tapfds[q] = -1;
> - }
> - } else if (dev->backend_type ==
> - VIRTIO_USER_BACKEND_VHOST_VDPA) {
> + break;
> + case VIRTIO_USER_BACKEND_VHOST_VDPA:
> dev->ops = &virtio_ops_vdpa;
> - } else {
> + break;
> + default:
> PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path);
> return -1;
> }
>
> -
> if (dev->ops->setup(dev) < 0) {
> PMD_INIT_LOG(ERR, "(%s) Failed to setup backend\n", dev->path);
> return -1;
> @@ -592,15 +574,6 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
> close(dev->callfds[i]);
> close(dev->kickfds[i]);
> }
> - if (dev->vhostfds) {
> - for (i = 0; i < dev->max_queue_pairs; ++i) {
> - close(dev->vhostfds[i]);
> - if (dev->tapfds[i] >= 0)
> - close(dev->tapfds[i]);
> - }
> - free(dev->vhostfds);
> - free(dev->tapfds);
> - }
>
> free(dev->ifname);
>
> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.h
> b/drivers/net/virtio/virtio_user/virtio_user_dev.h
> index a429dcc57c..5a2c9d38dd 100644
> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.h
> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.h
> @@ -31,11 +31,6 @@ struct virtio_user_dev {
> /* for vhost_vdpa backend */
> int vhostfd;
>
> - /* for vhost_kernel backend */
> - char *ifname;
> - int *vhostfds;
> - int *tapfds;
> -
> /* for both vhost_user and vhost_kernel */
> int callfds[VIRTIO_MAX_VIRTQUEUES];
> int kickfds[VIRTIO_MAX_VIRTQUEUES];
> @@ -55,6 +50,8 @@ struct virtio_user_dev {
> uint16_t port_id;
> uint8_t mac_addr[RTE_ETHER_ADDR_LEN];
> char path[PATH_MAX];
> + char *ifname;
> +
> union {
> struct vring vrings[VIRTIO_MAX_VIRTQUEUES];
> struct vring_packed packed_vrings[VIRTIO_MAX_VIRTQUEUES];
> --
> 2.29.2
On 1/11/21 9:02 AM, Xia, Chenbo wrote:
> Hi Maxime,
>
>> -----Original Message-----
>> From: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Sent: Monday, December 21, 2020 5:14 AM
>> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>; olivier.matz@6wind.com;
>> amorenoz@redhat.com; david.marchand@redhat.com
>> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Subject: [PATCH 39/40] net/virtio: move Vhost-kernel data to its backend
>>
>> As done earlier for Vhost-user, this patch moves the
>> Vhost-Kernel specific data to its backend file.
>>
>> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>> ---
>> drivers/net/virtio/virtio_user/vhost_kernel.c | 78 +++++++++++++++----
>> .../net/virtio/virtio_user/virtio_user_dev.c | 43 ++--------
>> .../net/virtio/virtio_user/virtio_user_dev.h | 7 +-
>> 3 files changed, 72 insertions(+), 56 deletions(-)
>
> <snip>
>
>> vhost_kernel_setup(struct virtio_user_dev *dev)
>> {
>> int vhostfd;
>> - uint32_t i;
>> + uint32_t q, i;
>> + struct vhost_kernel_data *data;
>> +
>> + data = malloc(sizeof(*data));
>> + if (!data) {
>> + PMD_INIT_LOG(ERR, "(%s) Failed to allocate Vhost-kernel data",
>> dev->path);
>> + return -1;
>> + }
>> +
>> + data->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
>> + data->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
>> + if (!data->vhostfds || !data->tapfds) {
>> + PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
>> + return -1;
>> + }
>> +
>> + for (q = 0; q < dev->max_queue_pairs; ++q) {
>> + data->vhostfds[q] = -1;
>> + data->tapfds[q] = -1;
>> + }
>
> Forget to do 'dev->backend_data = data'? 😊
Indeed! As I mentionned in the cover letter, I didn't test last 5
patches... and we can see it confirmed right here :)
I have reworked the fucntion to handle error properly (free allocated
resources, close opened FDs...)
>
> Besides,
>
> When I do test 'virtio_user for exceptional path', iperf just cannot normally
> send/recv pkts. Testpmd have some pkts (<10), guess tcp connection cannot be
> established. I have not tried to locate the root cause. Just thinking it's
> better to tell you first.
Thanks for the heads-up, I will look into it before posting v2.
Maxime
> Thanks!
> Chenbo
>
>>
>> get_vhost_kernel_max_regions();
>>
>> for (i = 0; i < dev->max_queue_pairs; ++i) {
>> vhostfd = open(dev->path, O_RDWR);
>> if (vhostfd < 0) {
>> - PMD_DRV_LOG(ERR, "fail to open %s, %s",
>> - dev->path, strerror(errno));
>> + PMD_DRV_LOG(ERR, "fail to open %s, %s", dev->path,
>> strerror(errno));
>> return -1;
>> }
>>
>> - dev->vhostfds[i] = vhostfd;
>> + data->vhostfds[i] = vhostfd;
>> }
>>
>> return 0;
>> @@ -360,6 +392,19 @@ vhost_kernel_setup(struct virtio_user_dev *dev)
>> static int
>> vhost_kernel_destroy(struct virtio_user_dev *dev)
>> {
>> + struct vhost_kernel_data *data = dev->backend_data;
>> + uint32_t i;
>> +
>> + for (i = 0; i < dev->max_queue_pairs; ++i) {
>> + close(data->vhostfds[i]);
>> + if (data->tapfds[i] >= 0)
>> + close(data->tapfds[i]);
>> + }
>> +
>> + free(data->vhostfds);
>> + free(data->tapfds);
>> + free(data);
>> +
>> return 0;
>> }
>>
>> @@ -395,14 +440,15 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev
>> *dev,
>> int vhostfd;
>> int tapfd;
>> int req_mq = (dev->max_queue_pairs > 1);
>> + struct vhost_kernel_data *data = dev->backend_data;
>>
>> - vhostfd = dev->vhostfds[pair_idx];
>> + vhostfd = data->vhostfds[pair_idx];
>>
>> if (dev->qp_enabled[pair_idx] == enable)
>> return 0;
>>
>> if (!enable) {
>> - tapfd = dev->tapfds[pair_idx];
>> + tapfd = data->tapfds[pair_idx];
>> if (vhost_kernel_set_backend(vhostfd, -1) < 0) {
>> PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
>> return -1;
>> @@ -415,8 +461,8 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
>> return 0;
>> }
>>
>> - if (dev->tapfds[pair_idx] >= 0) {
>> - tapfd = dev->tapfds[pair_idx];
>> + if (data->tapfds[pair_idx] >= 0) {
>> + tapfd = data->tapfds[pair_idx];
>> if (vhost_kernel_tap_set_offload(tapfd, dev->features) == -1)
>> return -1;
>> if (req_mq && vhost_kernel_tap_set_queue(tapfd, true) < 0) {
>> @@ -439,7 +485,7 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
>> return -1;
>> }
>>
>> - dev->tapfds[pair_idx] = tapfd;
>> + data->tapfds[pair_idx] = tapfd;
>>
>> set_backend:
>> if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
>> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> b/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> index 93a9ce2cd2..154aecc209 100644
>> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> @@ -397,11 +397,6 @@ virtio_user_mem_event_cb(enum rte_mem_event type
>> __rte_unused,
>> static int
>> virtio_user_dev_setup(struct virtio_user_dev *dev)
>> {
>> - uint32_t q;
>> -
>> - dev->vhostfds = NULL;
>> - dev->tapfds = NULL;
>> -
>> if (dev->is_server) {
>> if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) {
>> PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!");
>> @@ -409,34 +404,21 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
>> }
>> }
>>
>> - if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) {
>> + switch (dev->backend_type) {
>> + case VIRTIO_USER_BACKEND_VHOST_USER:
>> dev->ops = &virtio_ops_user;
>> - } else if (dev->backend_type ==
>> - VIRTIO_USER_BACKEND_VHOST_KERNEL) {
>> + break;
>> + case VIRTIO_USER_BACKEND_VHOST_KERNEL:
>> dev->ops = &virtio_ops_kernel;
>> -
>> - dev->vhostfds = malloc(dev->max_queue_pairs *
>> - sizeof(int));
>> - dev->tapfds = malloc(dev->max_queue_pairs *
>> - sizeof(int));
>> - if (!dev->vhostfds || !dev->tapfds) {
>> - PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
>> - return -1;
>> - }
>> -
>> - for (q = 0; q < dev->max_queue_pairs; ++q) {
>> - dev->vhostfds[q] = -1;
>> - dev->tapfds[q] = -1;
>> - }
>> - } else if (dev->backend_type ==
>> - VIRTIO_USER_BACKEND_VHOST_VDPA) {
>> + break;
>> + case VIRTIO_USER_BACKEND_VHOST_VDPA:
>> dev->ops = &virtio_ops_vdpa;
>> - } else {
>> + break;
>> + default:
>> PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path);
>> return -1;
>> }
>>
>> -
>> if (dev->ops->setup(dev) < 0) {
>> PMD_INIT_LOG(ERR, "(%s) Failed to setup backend\n", dev->path);
>> return -1;
>> @@ -592,15 +574,6 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
>> close(dev->callfds[i]);
>> close(dev->kickfds[i]);
>> }
>> - if (dev->vhostfds) {
>> - for (i = 0; i < dev->max_queue_pairs; ++i) {
>> - close(dev->vhostfds[i]);
>> - if (dev->tapfds[i] >= 0)
>> - close(dev->tapfds[i]);
>> - }
>> - free(dev->vhostfds);
>> - free(dev->tapfds);
>> - }
>>
>> free(dev->ifname);
>>
>> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.h
>> b/drivers/net/virtio/virtio_user/virtio_user_dev.h
>> index a429dcc57c..5a2c9d38dd 100644
>> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.h
>> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.h
>> @@ -31,11 +31,6 @@ struct virtio_user_dev {
>> /* for vhost_vdpa backend */
>> int vhostfd;
>>
>> - /* for vhost_kernel backend */
>> - char *ifname;
>> - int *vhostfds;
>> - int *tapfds;
>> -
>> /* for both vhost_user and vhost_kernel */
>> int callfds[VIRTIO_MAX_VIRTQUEUES];
>> int kickfds[VIRTIO_MAX_VIRTQUEUES];
>> @@ -55,6 +50,8 @@ struct virtio_user_dev {
>> uint16_t port_id;
>> uint8_t mac_addr[RTE_ETHER_ADDR_LEN];
>> char path[PATH_MAX];
>> + char *ifname;
>> +
>> union {
>> struct vring vrings[VIRTIO_MAX_VIRTQUEUES];
>> struct vring_packed packed_vrings[VIRTIO_MAX_VIRTQUEUES];
>> --
>> 2.29.2
>
On 1/11/21 9:02 AM, Xia, Chenbo wrote:
> Hi Maxime,
>
>> -----Original Message-----
>> From: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Sent: Monday, December 21, 2020 5:14 AM
>> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>; olivier.matz@6wind.com;
>> amorenoz@redhat.com; david.marchand@redhat.com
>> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Subject: [PATCH 39/40] net/virtio: move Vhost-kernel data to its backend
>>
>> As done earlier for Vhost-user, this patch moves the
>> Vhost-Kernel specific data to its backend file.
>>
>> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>> ---
>> drivers/net/virtio/virtio_user/vhost_kernel.c | 78 +++++++++++++++----
>> .../net/virtio/virtio_user/virtio_user_dev.c | 43 ++--------
>> .../net/virtio/virtio_user/virtio_user_dev.h | 7 +-
>> 3 files changed, 72 insertions(+), 56 deletions(-)
>
> <snip>
>
>> vhost_kernel_setup(struct virtio_user_dev *dev)
>> {
>> int vhostfd;
>> - uint32_t i;
>> + uint32_t q, i;
>> + struct vhost_kernel_data *data;
>> +
>> + data = malloc(sizeof(*data));
>> + if (!data) {
>> + PMD_INIT_LOG(ERR, "(%s) Failed to allocate Vhost-kernel data",
>> dev->path);
>> + return -1;
>> + }
>> +
>> + data->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
>> + data->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
>> + if (!data->vhostfds || !data->tapfds) {
>> + PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
>> + return -1;
>> + }
>> +
>> + for (q = 0; q < dev->max_queue_pairs; ++q) {
>> + data->vhostfds[q] = -1;
>> + data->tapfds[q] = -1;
>> + }
>
> Forget to do 'dev->backend_data = data'? 😊
>
> Besides,
>
> When I do test 'virtio_user for exceptional path', iperf just cannot normally
> send/recv pkts. Testpmd have some pkts (<10), guess tcp connection cannot be
> established. I have not tried to locate the root cause. Just thinking it's
> better to tell you first.
Thanks for reporting.
I did a git bisect and could find the issue:
static int
vhost_kernel_set_vring_kick(struct virtio_user_dev *dev, struct
vhost_vring_file *file)
{
return vhost_kernel_set_vring_file(dev, VHOST_SET_VRING_CALL, file);
}
static int
vhost_kernel_set_vring_call(struct virtio_user_dev *dev, struct
vhost_vring_file *file)
{
return vhost_kernel_set_vring_file(dev, VHOST_SET_VRING_KICK, file);
}
The kick and call request were inverted!
Once fixed, I can txonly without getting stuck once queue is full.
I will run more Vhost-kernel tests, but I think that was the issue you
found.
Maxime
> Thanks!
> Chenbo
>
>>
>> get_vhost_kernel_max_regions();
>>
>> for (i = 0; i < dev->max_queue_pairs; ++i) {
>> vhostfd = open(dev->path, O_RDWR);
>> if (vhostfd < 0) {
>> - PMD_DRV_LOG(ERR, "fail to open %s, %s",
>> - dev->path, strerror(errno));
>> + PMD_DRV_LOG(ERR, "fail to open %s, %s", dev->path,
>> strerror(errno));
>> return -1;
>> }
>>
>> - dev->vhostfds[i] = vhostfd;
>> + data->vhostfds[i] = vhostfd;
>> }
>>
>> return 0;
>> @@ -360,6 +392,19 @@ vhost_kernel_setup(struct virtio_user_dev *dev)
>> static int
>> vhost_kernel_destroy(struct virtio_user_dev *dev)
>> {
>> + struct vhost_kernel_data *data = dev->backend_data;
>> + uint32_t i;
>> +
>> + for (i = 0; i < dev->max_queue_pairs; ++i) {
>> + close(data->vhostfds[i]);
>> + if (data->tapfds[i] >= 0)
>> + close(data->tapfds[i]);
>> + }
>> +
>> + free(data->vhostfds);
>> + free(data->tapfds);
>> + free(data);
>> +
>> return 0;
>> }
>>
>> @@ -395,14 +440,15 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev
>> *dev,
>> int vhostfd;
>> int tapfd;
>> int req_mq = (dev->max_queue_pairs > 1);
>> + struct vhost_kernel_data *data = dev->backend_data;
>>
>> - vhostfd = dev->vhostfds[pair_idx];
>> + vhostfd = data->vhostfds[pair_idx];
>>
>> if (dev->qp_enabled[pair_idx] == enable)
>> return 0;
>>
>> if (!enable) {
>> - tapfd = dev->tapfds[pair_idx];
>> + tapfd = data->tapfds[pair_idx];
>> if (vhost_kernel_set_backend(vhostfd, -1) < 0) {
>> PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
>> return -1;
>> @@ -415,8 +461,8 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
>> return 0;
>> }
>>
>> - if (dev->tapfds[pair_idx] >= 0) {
>> - tapfd = dev->tapfds[pair_idx];
>> + if (data->tapfds[pair_idx] >= 0) {
>> + tapfd = data->tapfds[pair_idx];
>> if (vhost_kernel_tap_set_offload(tapfd, dev->features) == -1)
>> return -1;
>> if (req_mq && vhost_kernel_tap_set_queue(tapfd, true) < 0) {
>> @@ -439,7 +485,7 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
>> return -1;
>> }
>>
>> - dev->tapfds[pair_idx] = tapfd;
>> + data->tapfds[pair_idx] = tapfd;
>>
>> set_backend:
>> if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
>> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> b/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> index 93a9ce2cd2..154aecc209 100644
>> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> @@ -397,11 +397,6 @@ virtio_user_mem_event_cb(enum rte_mem_event type
>> __rte_unused,
>> static int
>> virtio_user_dev_setup(struct virtio_user_dev *dev)
>> {
>> - uint32_t q;
>> -
>> - dev->vhostfds = NULL;
>> - dev->tapfds = NULL;
>> -
>> if (dev->is_server) {
>> if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) {
>> PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!");
>> @@ -409,34 +404,21 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
>> }
>> }
>>
>> - if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) {
>> + switch (dev->backend_type) {
>> + case VIRTIO_USER_BACKEND_VHOST_USER:
>> dev->ops = &virtio_ops_user;
>> - } else if (dev->backend_type ==
>> - VIRTIO_USER_BACKEND_VHOST_KERNEL) {
>> + break;
>> + case VIRTIO_USER_BACKEND_VHOST_KERNEL:
>> dev->ops = &virtio_ops_kernel;
>> -
>> - dev->vhostfds = malloc(dev->max_queue_pairs *
>> - sizeof(int));
>> - dev->tapfds = malloc(dev->max_queue_pairs *
>> - sizeof(int));
>> - if (!dev->vhostfds || !dev->tapfds) {
>> - PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
>> - return -1;
>> - }
>> -
>> - for (q = 0; q < dev->max_queue_pairs; ++q) {
>> - dev->vhostfds[q] = -1;
>> - dev->tapfds[q] = -1;
>> - }
>> - } else if (dev->backend_type ==
>> - VIRTIO_USER_BACKEND_VHOST_VDPA) {
>> + break;
>> + case VIRTIO_USER_BACKEND_VHOST_VDPA:
>> dev->ops = &virtio_ops_vdpa;
>> - } else {
>> + break;
>> + default:
>> PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path);
>> return -1;
>> }
>>
>> -
>> if (dev->ops->setup(dev) < 0) {
>> PMD_INIT_LOG(ERR, "(%s) Failed to setup backend\n", dev->path);
>> return -1;
>> @@ -592,15 +574,6 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
>> close(dev->callfds[i]);
>> close(dev->kickfds[i]);
>> }
>> - if (dev->vhostfds) {
>> - for (i = 0; i < dev->max_queue_pairs; ++i) {
>> - close(dev->vhostfds[i]);
>> - if (dev->tapfds[i] >= 0)
>> - close(dev->tapfds[i]);
>> - }
>> - free(dev->vhostfds);
>> - free(dev->tapfds);
>> - }
>>
>> free(dev->ifname);
>>
>> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.h
>> b/drivers/net/virtio/virtio_user/virtio_user_dev.h
>> index a429dcc57c..5a2c9d38dd 100644
>> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.h
>> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.h
>> @@ -31,11 +31,6 @@ struct virtio_user_dev {
>> /* for vhost_vdpa backend */
>> int vhostfd;
>>
>> - /* for vhost_kernel backend */
>> - char *ifname;
>> - int *vhostfds;
>> - int *tapfds;
>> -
>> /* for both vhost_user and vhost_kernel */
>> int callfds[VIRTIO_MAX_VIRTQUEUES];
>> int kickfds[VIRTIO_MAX_VIRTQUEUES];
>> @@ -55,6 +50,8 @@ struct virtio_user_dev {
>> uint16_t port_id;
>> uint8_t mac_addr[RTE_ETHER_ADDR_LEN];
>> char path[PATH_MAX];
>> + char *ifname;
>> +
>> union {
>> struct vring vrings[VIRTIO_MAX_VIRTQUEUES];
>> struct vring_packed packed_vrings[VIRTIO_MAX_VIRTQUEUES];
>> --
>> 2.29.2
>
@@ -14,6 +14,11 @@
#include "virtio_user_dev.h"
#include "vhost_kernel_tap.h"
+struct vhost_kernel_data {
+ int *vhostfds;
+ int *tapfds;
+};
+
struct vhost_memory_kernel {
uint32_t nregions;
uint32_t padding;
@@ -96,7 +101,9 @@ vhost_kernel_ioctl(int fd, uint64_t request, void *arg)
static int
vhost_kernel_set_owner(struct virtio_user_dev *dev)
{
- return vhost_kernel_ioctl(dev->vhostfds[0], VHOST_SET_OWNER, NULL);
+ struct vhost_kernel_data *data = dev->backend_data;
+
+ return vhost_kernel_ioctl(data->vhostfds[0], VHOST_SET_OWNER, NULL);
}
static int
@@ -104,8 +111,9 @@ vhost_kernel_get_features(struct virtio_user_dev *dev, uint64_t *features)
{
int ret;
unsigned int tap_features;
+ struct vhost_kernel_data *data = dev->backend_data;
- ret = vhost_kernel_ioctl(dev->vhostfds[0], VHOST_GET_FEATURES, features);
+ ret = vhost_kernel_ioctl(data->vhostfds[0], VHOST_GET_FEATURES, features);
if (ret < 0) {
PMD_DRV_LOG(ERR, "Failed to get features");
return -1;
@@ -138,6 +146,8 @@ vhost_kernel_get_features(struct virtio_user_dev *dev, uint64_t *features)
static int
vhost_kernel_set_features(struct virtio_user_dev *dev, uint64_t features)
{
+ struct vhost_kernel_data *data = dev->backend_data;
+
/* We don't need memory protection here */
features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
/* VHOST kernel does not know about below flags */
@@ -145,7 +155,7 @@ vhost_kernel_set_features(struct virtio_user_dev *dev, uint64_t features)
features &= ~VHOST_KERNEL_HOST_OFFLOADS_MASK;
features &= ~(1ULL << VIRTIO_NET_F_MQ);
- return vhost_kernel_ioctl(dev->vhostfds[0], VHOST_SET_FEATURES, &features);
+ return vhost_kernel_ioctl(data->vhostfds[0], VHOST_SET_FEATURES, &features);
}
static int
@@ -185,6 +195,7 @@ add_memseg_list(const struct rte_memseg_list *msl, void *arg)
static int
vhost_kernel_set_memory_table(struct virtio_user_dev *dev)
{
+ struct vhost_kernel_data *data = dev->backend_data;
struct vhost_memory_kernel *vm;
int ret;
@@ -205,7 +216,7 @@ vhost_kernel_set_memory_table(struct virtio_user_dev *dev)
if (ret < 0)
goto err_free;
- ret = vhost_kernel_ioctl(dev->vhostfds[0], VHOST_SET_MEM_TABLE, vm);
+ ret = vhost_kernel_ioctl(data->vhostfds[0], VHOST_SET_MEM_TABLE, vm);
if (ret < 0)
goto err_free;
@@ -224,9 +235,10 @@ vhost_kernel_set_vring(struct virtio_user_dev *dev, uint64_t req, struct vhost_v
{
int ret, fd;
uint32_t index = state->index;
+ struct vhost_kernel_data *data = dev->backend_data;
/* Convert from queue index to queue-pair & offset */
- fd = dev->vhostfds[state->index / 2];
+ fd = data->vhostfds[state->index / 2];
state->index %= 2;
ret = vhost_kernel_ioctl(fd, req, state);
@@ -265,9 +277,10 @@ vhost_kernel_set_vring_file(struct virtio_user_dev *dev, uint64_t req,
{
int ret, fd;
uint32_t index = file->index;
+ struct vhost_kernel_data *data = dev->backend_data;
/* Convert from queue index to queue-pair & offset */
- fd = dev->vhostfds[file->index / 2];
+ fd = data->vhostfds[file->index / 2];
file->index %= 2;
ret = vhost_kernel_ioctl(fd, req, file);
@@ -299,9 +312,10 @@ vhost_kernel_set_vring_addr(struct virtio_user_dev *dev, struct vhost_vring_addr
{
int ret, fd;
uint32_t index = addr->index;
+ struct vhost_kernel_data *data = dev->backend_data;
/* Convert from queue index to queue-pair & offset */
- fd = dev->vhostfds[addr->index / 2];
+ fd = data->vhostfds[addr->index / 2];
addr->index %= 2;
ret = vhost_kernel_ioctl(fd, VHOST_SET_VRING_ADDR, addr);
@@ -339,19 +353,37 @@ static int
vhost_kernel_setup(struct virtio_user_dev *dev)
{
int vhostfd;
- uint32_t i;
+ uint32_t q, i;
+ struct vhost_kernel_data *data;
+
+ data = malloc(sizeof(*data));
+ if (!data) {
+ PMD_INIT_LOG(ERR, "(%s) Failed to allocate Vhost-kernel data", dev->path);
+ return -1;
+ }
+
+ data->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
+ data->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
+ if (!data->vhostfds || !data->tapfds) {
+ PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
+ return -1;
+ }
+
+ for (q = 0; q < dev->max_queue_pairs; ++q) {
+ data->vhostfds[q] = -1;
+ data->tapfds[q] = -1;
+ }
get_vhost_kernel_max_regions();
for (i = 0; i < dev->max_queue_pairs; ++i) {
vhostfd = open(dev->path, O_RDWR);
if (vhostfd < 0) {
- PMD_DRV_LOG(ERR, "fail to open %s, %s",
- dev->path, strerror(errno));
+ PMD_DRV_LOG(ERR, "fail to open %s, %s", dev->path, strerror(errno));
return -1;
}
- dev->vhostfds[i] = vhostfd;
+ data->vhostfds[i] = vhostfd;
}
return 0;
@@ -360,6 +392,19 @@ vhost_kernel_setup(struct virtio_user_dev *dev)
static int
vhost_kernel_destroy(struct virtio_user_dev *dev)
{
+ struct vhost_kernel_data *data = dev->backend_data;
+ uint32_t i;
+
+ for (i = 0; i < dev->max_queue_pairs; ++i) {
+ close(data->vhostfds[i]);
+ if (data->tapfds[i] >= 0)
+ close(data->tapfds[i]);
+ }
+
+ free(data->vhostfds);
+ free(data->tapfds);
+ free(data);
+
return 0;
}
@@ -395,14 +440,15 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
int vhostfd;
int tapfd;
int req_mq = (dev->max_queue_pairs > 1);
+ struct vhost_kernel_data *data = dev->backend_data;
- vhostfd = dev->vhostfds[pair_idx];
+ vhostfd = data->vhostfds[pair_idx];
if (dev->qp_enabled[pair_idx] == enable)
return 0;
if (!enable) {
- tapfd = dev->tapfds[pair_idx];
+ tapfd = data->tapfds[pair_idx];
if (vhost_kernel_set_backend(vhostfd, -1) < 0) {
PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
return -1;
@@ -415,8 +461,8 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
return 0;
}
- if (dev->tapfds[pair_idx] >= 0) {
- tapfd = dev->tapfds[pair_idx];
+ if (data->tapfds[pair_idx] >= 0) {
+ tapfd = data->tapfds[pair_idx];
if (vhost_kernel_tap_set_offload(tapfd, dev->features) == -1)
return -1;
if (req_mq && vhost_kernel_tap_set_queue(tapfd, true) < 0) {
@@ -439,7 +485,7 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
return -1;
}
- dev->tapfds[pair_idx] = tapfd;
+ data->tapfds[pair_idx] = tapfd;
set_backend:
if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
@@ -397,11 +397,6 @@ virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused,
static int
virtio_user_dev_setup(struct virtio_user_dev *dev)
{
- uint32_t q;
-
- dev->vhostfds = NULL;
- dev->tapfds = NULL;
-
if (dev->is_server) {
if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) {
PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!");
@@ -409,34 +404,21 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
}
}
- if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) {
+ switch (dev->backend_type) {
+ case VIRTIO_USER_BACKEND_VHOST_USER:
dev->ops = &virtio_ops_user;
- } else if (dev->backend_type ==
- VIRTIO_USER_BACKEND_VHOST_KERNEL) {
+ break;
+ case VIRTIO_USER_BACKEND_VHOST_KERNEL:
dev->ops = &virtio_ops_kernel;
-
- dev->vhostfds = malloc(dev->max_queue_pairs *
- sizeof(int));
- dev->tapfds = malloc(dev->max_queue_pairs *
- sizeof(int));
- if (!dev->vhostfds || !dev->tapfds) {
- PMD_INIT_LOG(ERR, "(%s) Failed to allocate FDs", dev->path);
- return -1;
- }
-
- for (q = 0; q < dev->max_queue_pairs; ++q) {
- dev->vhostfds[q] = -1;
- dev->tapfds[q] = -1;
- }
- } else if (dev->backend_type ==
- VIRTIO_USER_BACKEND_VHOST_VDPA) {
+ break;
+ case VIRTIO_USER_BACKEND_VHOST_VDPA:
dev->ops = &virtio_ops_vdpa;
- } else {
+ break;
+ default:
PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path);
return -1;
}
-
if (dev->ops->setup(dev) < 0) {
PMD_INIT_LOG(ERR, "(%s) Failed to setup backend\n", dev->path);
return -1;
@@ -592,15 +574,6 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
close(dev->callfds[i]);
close(dev->kickfds[i]);
}
- if (dev->vhostfds) {
- for (i = 0; i < dev->max_queue_pairs; ++i) {
- close(dev->vhostfds[i]);
- if (dev->tapfds[i] >= 0)
- close(dev->tapfds[i]);
- }
- free(dev->vhostfds);
- free(dev->tapfds);
- }
free(dev->ifname);
@@ -31,11 +31,6 @@ struct virtio_user_dev {
/* for vhost_vdpa backend */
int vhostfd;
- /* for vhost_kernel backend */
- char *ifname;
- int *vhostfds;
- int *tapfds;
-
/* for both vhost_user and vhost_kernel */
int callfds[VIRTIO_MAX_VIRTQUEUES];
int kickfds[VIRTIO_MAX_VIRTQUEUES];
@@ -55,6 +50,8 @@ struct virtio_user_dev {
uint16_t port_id;
uint8_t mac_addr[RTE_ETHER_ADDR_LEN];
char path[PATH_MAX];
+ char *ifname;
+
union {
struct vring vrings[VIRTIO_MAX_VIRTQUEUES];
struct vring_packed packed_vrings[VIRTIO_MAX_VIRTQUEUES];