[v3,3/5] vdpa/mlx5: workaround dirty bitmap MR creation
Checks
Commit Message
Due to kernel driver/FW issues in direct MKEY creation using the DevX
API, this patch replaces the dirty bitmap MR creation to use wrapped
mkey instead.
Fixes: 9d39e57f21ac ("vdpa/mlx5: support live migration")
Cc: stable@dpdk.org
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
drivers/vdpa/mlx5/mlx5_vdpa.h | 1 +
drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 38 +++++++------------------------
drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 2 ++
3 files changed, 11 insertions(+), 30 deletions(-)
@@ -147,6 +147,7 @@ struct mlx5_vdpa_priv {
struct mlx5_vdpa_steer steer;
struct mlx5dv_var *var;
void *virtq_db_addr;
+ struct mlx5_pmd_wrapped_mr lm_mr;
SLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list;
struct mlx5_vdpa_virtq virtqs[];
};
@@ -36,38 +36,22 @@ int
mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
uint64_t log_size)
{
- struct mlx5_devx_mkey_attr mkey_attr = {
- .addr = (uintptr_t)log_base,
- .size = log_size,
- .pd = priv->cdev->pdn,
- .pg_access = 1,
- };
struct mlx5_devx_virtq_attr attr = {
.type = MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS,
.dirty_bitmap_addr = log_base,
.dirty_bitmap_size = log_size,
};
- struct mlx5_vdpa_query_mr *mr = rte_malloc(__func__, sizeof(*mr), 0);
int i;
+ int ret = mlx5_os_wrapped_mkey_create(priv->cdev->ctx, priv->cdev->pd,
+ priv->cdev->pdn,
+ (void *)(uintptr_t)log_base,
+ log_size, &priv->lm_mr);
- if (!mr) {
- DRV_LOG(ERR, "Failed to allocate mem for lm mr.");
+ if (!ret) {
+ DRV_LOG(ERR, "Failed to allocate wrapped MR for lm.");
return -1;
}
- mr->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
- (void *)(uintptr_t)log_base,
- log_size, IBV_ACCESS_LOCAL_WRITE);
- if (!mr->umem) {
- DRV_LOG(ERR, "Failed to register umem for lm mr.");
- goto err;
- }
- mkey_attr.umem_id = mr->umem->umem_id;
- mr->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &mkey_attr);
- if (!mr->mkey) {
- DRV_LOG(ERR, "Failed to create Mkey for lm.");
- goto err;
- }
- attr.dirty_bitmap_mkey = mr->mkey->id;
+ attr.dirty_bitmap_mkey = priv->lm_mr.lkey;
for (i = 0; i < priv->nr_virtqs; ++i) {
attr.queue_index = i;
if (!priv->virtqs[i].virtq) {
@@ -78,15 +62,9 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
goto err;
}
}
- mr->is_indirect = 0;
- SLIST_INSERT_HEAD(&priv->mr_list, mr, next);
return 0;
err:
- if (mr->mkey)
- mlx5_devx_cmd_destroy(mr->mkey);
- if (mr->umem)
- mlx5_glue->devx_umem_dereg(mr->umem);
- rte_free(mr);
+ mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
return -1;
}
@@ -31,6 +31,8 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
entry = next;
}
SLIST_INIT(&priv->mr_list);
+ if (priv->lm_mr.addr)
+ mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
if (priv->null_mr) {
claim_zero(mlx5_glue->dereg_mr(priv->null_mr));
priv->null_mr = NULL;