@@ -275,13 +275,17 @@ mlx5_vdpa_wait_dev_close_tasks_done(struct mlx5_vdpa_priv *priv)
}
static int
-mlx5_vdpa_dev_close(int vid)
+_internal_mlx5_vdpa_dev_close(int vid, bool release_resource)
{
struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
- struct mlx5_vdpa_priv *priv =
- mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+ struct mlx5_vdpa_priv *priv;
int ret = 0;
+ if (!vdev) {
+ DRV_LOG(ERR, "Invalid vDPA device.");
+ return -1;
+ }
+ priv = mlx5_vdpa_find_priv_resource_by_vdev(vdev);
if (priv == NULL) {
DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
return -1;
@@ -291,7 +295,7 @@ mlx5_vdpa_dev_close(int vid)
ret |= mlx5_vdpa_lm_log(priv);
priv->state = MLX5_VDPA_STATE_IN_PROGRESS;
}
- if (priv->use_c_thread) {
+ if (priv->use_c_thread && !release_resource) {
if (priv->last_c_thrd_idx >=
(conf_thread_mng.max_thrds - 1))
priv->last_c_thrd_idx = 0;
@@ -315,7 +319,7 @@ mlx5_vdpa_dev_close(int vid)
pthread_mutex_lock(&priv->steer_update_lock);
mlx5_vdpa_steer_unset(priv);
pthread_mutex_unlock(&priv->steer_update_lock);
- mlx5_vdpa_virtqs_release(priv);
+ mlx5_vdpa_virtqs_release(priv, release_resource);
mlx5_vdpa_drain_cq(priv);
if (priv->lm_mr.addr)
mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
@@ -329,6 +333,12 @@ mlx5_vdpa_dev_close(int vid)
return ret;
}
+static int
+mlx5_vdpa_dev_close(int vid)
+{
+ return _internal_mlx5_vdpa_dev_close(vid, false);
+}
+
static int
mlx5_vdpa_dev_config(int vid)
{
@@ -624,8 +634,9 @@ mlx5_vdpa_config_get(struct mlx5_kvargs_ctrl *mkvlist,
static int
mlx5_vdpa_virtq_resource_prepare(struct mlx5_vdpa_priv *priv)
{
+ uint32_t remaining_cnt = 0, err_cnt = 0, task_num = 0;
uint32_t max_queues = priv->queues * 2;
- uint32_t index;
+ uint32_t index, thrd_idx, data[1];
struct mlx5_vdpa_virtq *virtq;
for (index = 0; index < priv->caps.max_num_virtio_queues * 2;
@@ -635,10 +646,48 @@ mlx5_vdpa_virtq_resource_prepare(struct mlx5_vdpa_priv *priv)
}
if (!priv->queues || !priv->queue_size)
return 0;
- for (index = 0; index < max_queues; ++index)
- if (mlx5_vdpa_virtq_single_resource_prepare(priv,
- index))
+ if (priv->use_c_thread) {
+ uint32_t main_task_idx[max_queues];
+
+ for (index = 0; index < max_queues; ++index) {
+ virtq = &priv->virtqs[index];
+ thrd_idx = index % (conf_thread_mng.max_thrds + 1);
+ if (!thrd_idx) {
+ main_task_idx[task_num] = index;
+ task_num++;
+ continue;
+ }
+ thrd_idx = priv->last_c_thrd_idx + 1;
+ if (thrd_idx >= conf_thread_mng.max_thrds)
+ thrd_idx = 0;
+ priv->last_c_thrd_idx = thrd_idx;
+ data[0] = index;
+ if (mlx5_vdpa_task_add(priv, thrd_idx,
+ MLX5_VDPA_TASK_PREPARE_VIRTQ,
+ &remaining_cnt, &err_cnt,
+ (void **)&data, 1)) {
+ DRV_LOG(ERR, "Fail to add "
+ "task prepare virtq (%d).", index);
+ main_task_idx[task_num] = index;
+ task_num++;
+ }
+ }
+ for (index = 0; index < task_num; ++index)
+ if (mlx5_vdpa_virtq_single_resource_prepare(priv,
+ main_task_idx[index]))
+ goto error;
+ if (mlx5_vdpa_c_thread_wait_bulk_tasks_done(&remaining_cnt,
+ &err_cnt, 2000)) {
+ DRV_LOG(ERR,
+ "Failed to wait virt-queue prepare tasks ready.");
goto error;
+ }
+ } else {
+ for (index = 0; index < max_queues; ++index)
+ if (mlx5_vdpa_virtq_single_resource_prepare(priv,
+ index))
+ goto error;
+ }
if (mlx5_vdpa_is_modify_virtq_supported(priv))
if (mlx5_vdpa_steer_update(priv, true))
goto error;
@@ -855,7 +904,7 @@ static void
mlx5_vdpa_dev_release(struct mlx5_vdpa_priv *priv)
{
if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
- mlx5_vdpa_dev_close(priv->vid);
+ _internal_mlx5_vdpa_dev_close(priv->vid, true);
if (priv->use_c_thread)
mlx5_vdpa_wait_dev_close_tasks_done(priv);
mlx5_vdpa_release_dev_resources(priv);
@@ -85,6 +85,7 @@ enum mlx5_vdpa_task_type {
MLX5_VDPA_TASK_SETUP_VIRTQ,
MLX5_VDPA_TASK_STOP_VIRTQ,
MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT,
+ MLX5_VDPA_TASK_PREPARE_VIRTQ,
};
/* Generic task information and size must be multiple of 4B. */
@@ -355,8 +356,12 @@ void mlx5_vdpa_err_event_unset(struct mlx5_vdpa_priv *priv);
*
* @param[in] priv
* The vdpa driver private structure.
+ * @param[in] release_resource
+ * The vdpa driver realease resource without prepare resource.
*/
-void mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv);
+void
+mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv,
+ bool release_resource);
/**
* Cleanup cached resources of all virtqs.
@@ -196,7 +196,7 @@ mlx5_vdpa_c_thread_handle(void *arg)
pthread_mutex_lock(&priv->steer_update_lock);
mlx5_vdpa_steer_unset(priv);
pthread_mutex_unlock(&priv->steer_update_lock);
- mlx5_vdpa_virtqs_release(priv);
+ mlx5_vdpa_virtqs_release(priv, false);
mlx5_vdpa_drain_cq(priv);
if (priv->lm_mr.addr)
mlx5_os_wrapped_mkey_destroy(
@@ -208,6 +208,18 @@ mlx5_vdpa_c_thread_handle(void *arg)
&priv->dev_close_progress, 0,
__ATOMIC_RELAXED);
break;
+ case MLX5_VDPA_TASK_PREPARE_VIRTQ:
+ ret = mlx5_vdpa_virtq_single_resource_prepare(
+ priv, task.idx);
+ if (ret) {
+ DRV_LOG(ERR,
+ "Failed to prepare virtq %d.",
+ task.idx);
+ __atomic_fetch_add(
+ task.err_cnt, 1,
+ __ATOMIC_RELAXED);
+ }
+ break;
default:
DRV_LOG(ERR, "Invalid vdpa task type %d.",
task.type);
@@ -113,6 +113,16 @@ mlx5_vdpa_virtq_unreg_intr_handle_all(struct mlx5_vdpa_priv *priv)
}
}
+static void
+mlx5_vdpa_vq_destroy(struct mlx5_vdpa_virtq *virtq)
+{
+ /* Clean pre-created resource in dev removal only */
+ claim_zero(mlx5_devx_cmd_destroy(virtq->virtq));
+ virtq->index = 0;
+ virtq->virtq = NULL;
+ virtq->configured = 0;
+}
+
/* Release cached VQ resources. */
void
mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv)
@@ -125,6 +135,8 @@ mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv)
if (virtq->index != i)
continue;
pthread_mutex_lock(&virtq->virtq_lock);
+ if (virtq->virtq)
+ mlx5_vdpa_vq_destroy(virtq);
for (j = 0; j < RTE_DIM(virtq->umems); ++j) {
if (virtq->umems[j].obj) {
claim_zero(mlx5_glue->devx_umem_dereg
@@ -154,29 +166,34 @@ mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq)
if (ret)
DRV_LOG(WARNING, "Failed to stop virtq %d.",
virtq->index);
- claim_zero(mlx5_devx_cmd_destroy(virtq->virtq));
- virtq->index = 0;
- virtq->virtq = NULL;
- virtq->configured = 0;
+ mlx5_vdpa_vq_destroy(virtq);
}
virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_DISABLED;
}
void
-mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv)
+mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv,
+ bool release_resource)
{
struct mlx5_vdpa_virtq *virtq;
- int i;
+ uint32_t i, max_virtq;
- for (i = 0; i < priv->nr_virtqs; i++) {
+ max_virtq = (release_resource &&
+ (priv->queues * 2) > priv->nr_virtqs) ?
+ (priv->queues * 2) : priv->nr_virtqs;
+ for (i = 0; i < max_virtq; i++) {
virtq = &priv->virtqs[i];
pthread_mutex_lock(&virtq->virtq_lock);
mlx5_vdpa_virtq_unset(virtq);
- if (i < (priv->queues * 2))
+ if (!release_resource && i < (priv->queues * 2))
mlx5_vdpa_virtq_single_resource_prepare(
priv, i);
pthread_mutex_unlock(&virtq->virtq_lock);
}
+ if (!release_resource && priv->queues &&
+ mlx5_vdpa_is_modify_virtq_supported(priv))
+ if (mlx5_vdpa_steer_update(priv, true))
+ mlx5_vdpa_steer_unset(priv);
priv->features = 0;
priv->nr_virtqs = 0;
}
@@ -733,7 +750,7 @@ mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)
}
return 0;
error:
- mlx5_vdpa_virtqs_release(priv);
+ mlx5_vdpa_virtqs_release(priv, true);
return -1;
}