[v2,3/7] vdpa/mlx5: no kick handling during shutdown
Checks
Commit Message
When Qemu suspend a VM, hw notifier is un-mmapped while vCPU thread may
still active and write notifier through kick socket.
PMD kick handler thread tries to install hw notifier through client
socket in such case will timeout and slow down device close.
This patch skips hw notifier install if VQ or device in middle of
shutdown.
Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
drivers/vdpa/mlx5/mlx5_vdpa.c | 17 ++++++++++-------
drivers/vdpa/mlx5/mlx5_vdpa.h | 8 +++++++-
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 12 +++++++++++-
3 files changed, 28 insertions(+), 9 deletions(-)
Comments
On 2/24/22 16:50, Xueming Li wrote:
> When Qemu suspend a VM, hw notifier is un-mmapped while vCPU thread may
suspends
> still active and write notifier through kick socket.
still be active
>
> PMD kick handler thread tries to install hw notifier through client
> socket in such case will timeout and slow down device close.
socket. In such case, it will
>
> This patch skips hw notifier install if VQ or device in middle of
> shutdown.
>
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> ---
> drivers/vdpa/mlx5/mlx5_vdpa.c | 17 ++++++++++-------
> drivers/vdpa/mlx5/mlx5_vdpa.h | 8 +++++++-
> drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 12 +++++++++++-
> 3 files changed, 28 insertions(+), 9 deletions(-)
>
Other than the commit messages comments:
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
If you are fine with my suggestions and no other revision needed, I can
fixup while applying.
Thanks,
Maxime
> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Wednesday, April 20, 2022 8:38 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Slava Ovsiienko <viacheslavo@nvidia.com>
> Subject: Re: [PATCH v2 3/7] vdpa/mlx5: no kick handling during shutdown
>
>
>
> On 2/24/22 16:50, Xueming Li wrote:
> > When Qemu suspend a VM, hw notifier is un-mmapped while vCPU thread
> > may
> suspends
> > still active and write notifier through kick socket.
> still be active
>
> >
> > PMD kick handler thread tries to install hw notifier through client
> > socket in such case will timeout and slow down device close.
> socket. In such case, it will
>
> >
> > This patch skips hw notifier install if VQ or device in middle of
> > shutdown.
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > ---
> > drivers/vdpa/mlx5/mlx5_vdpa.c | 17 ++++++++++-------
> > drivers/vdpa/mlx5/mlx5_vdpa.h | 8 +++++++-
> > drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 12 +++++++++++-
> > 3 files changed, 28 insertions(+), 9 deletions(-)
> >
>
> Other than the commit messages comments:
>
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>
> If you are fine with my suggestions and no other revision needed, I can fixup while applying.
Hi Maxime,
No further changes so far, please continue, thanks for taking care of this series!
>
> Thanks,
> Maxime
@@ -252,13 +252,15 @@ mlx5_vdpa_dev_close(int vid)
}
mlx5_vdpa_err_event_unset(priv);
mlx5_vdpa_cqe_event_unset(priv);
- if (priv->configured)
+ if (priv->state == MLX5_VDPA_STATE_CONFIGURED) {
ret |= mlx5_vdpa_lm_log(priv);
+ priv->state = MLX5_VDPA_STATE_IN_PROGRESS;
+ }
mlx5_vdpa_steer_unset(priv);
mlx5_vdpa_virtqs_release(priv);
mlx5_vdpa_event_qp_global_release(priv);
mlx5_vdpa_mem_dereg(priv);
- priv->configured = 0;
+ priv->state = MLX5_VDPA_STATE_PROBED;
priv->vid = 0;
/* The mutex may stay locked after event thread cancel - initiate it. */
pthread_mutex_init(&priv->vq_config_lock, NULL);
@@ -277,7 +279,8 @@ mlx5_vdpa_dev_config(int vid)
DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
return -EINVAL;
}
- if (priv->configured && mlx5_vdpa_dev_close(vid)) {
+ if (priv->state == MLX5_VDPA_STATE_CONFIGURED &&
+ mlx5_vdpa_dev_close(vid)) {
DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
return -1;
}
@@ -291,7 +294,7 @@ mlx5_vdpa_dev_config(int vid)
mlx5_vdpa_dev_close(vid);
return -1;
}
- priv->configured = 1;
+ priv->state = MLX5_VDPA_STATE_CONFIGURED;
DRV_LOG(INFO, "vDPA device %d was configured.", vid);
return 0;
}
@@ -373,7 +376,7 @@ mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
return -ENODEV;
}
- if (!priv->configured) {
+ if (priv->state == MLX5_VDPA_STATE_PROBED) {
DRV_LOG(ERR, "Device %s was not configured.",
vdev->device->name);
return -ENODATA;
@@ -401,7 +404,7 @@ mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
return -ENODEV;
}
- if (!priv->configured) {
+ if (priv->state == MLX5_VDPA_STATE_PROBED) {
DRV_LOG(ERR, "Device %s was not configured.",
vdev->device->name);
return -ENODATA;
@@ -590,7 +593,7 @@ mlx5_vdpa_dev_remove(struct mlx5_common_device *cdev)
TAILQ_REMOVE(&priv_list, priv, next);
pthread_mutex_unlock(&priv_list_lock);
if (found) {
- if (priv->configured)
+ if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
mlx5_vdpa_dev_close(priv->vid);
if (priv->var) {
mlx5_glue->dv_free_var(priv->var);
@@ -113,9 +113,15 @@ enum {
MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT
};
+enum mlx5_dev_state {
+ MLX5_VDPA_STATE_PROBED = 0,
+ MLX5_VDPA_STATE_CONFIGURED,
+ MLX5_VDPA_STATE_IN_PROGRESS /* Shutting down. */
+};
+
struct mlx5_vdpa_priv {
TAILQ_ENTRY(mlx5_vdpa_priv) next;
- uint8_t configured;
+ enum mlx5_dev_state state;
pthread_mutex_t vq_config_lock;
uint64_t no_traffic_counter;
pthread_t timer_tid;
@@ -25,6 +25,11 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg)
int nbytes;
int retry;
+ if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
+ DRV_LOG(ERR, "device %d queue %d down, skip kick handling",
+ priv->vid, virtq->index);
+ return;
+ }
if (rte_intr_fd_get(virtq->intr_handle) < 0)
return;
for (retry = 0; retry < 3; ++retry) {
@@ -43,6 +48,11 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg)
if (nbytes < 0)
return;
rte_write32(virtq->index, priv->virtq_db_addr);
+ if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
+ DRV_LOG(ERR, "device %d queue %d down, skip kick handling",
+ priv->vid, virtq->index);
+ return;
+ }
if (virtq->notifier_state == MLX5_VDPA_NOTIFIER_STATE_DISABLED) {
if (rte_vhost_host_notifier_ctrl(priv->vid, virtq->index, true))
virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_ERR;
@@ -541,7 +551,7 @@ mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable)
DRV_LOG(INFO, "Update virtq %d status %sable -> %sable.", index,
virtq->enable ? "en" : "dis", enable ? "en" : "dis");
- if (!priv->configured) {
+ if (priv->state == MLX5_VDPA_STATE_PROBED) {
virtq->enable = !!enable;
return 0;
}