Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/65298/?format=api
{ "id": 65298, "url": "http://patches.dpdk.org/api/patches/65298/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1580292549-27439-12-git-send-email-matan@mellanox.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1580292549-27439-12-git-send-email-matan@mellanox.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1580292549-27439-12-git-send-email-matan@mellanox.com", "date": "2020-01-29T10:09:07", "name": "[v2,11/13] vdpa/mlx5: support live migration", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "1f7656578a4af0cb5bd6ec2deafa87a15b8f2fb3", "submitter": { "id": 796, "url": "http://patches.dpdk.org/api/people/796/?format=api", "name": "Matan Azrad", "email": "matan@mellanox.com" }, "delegate": { "id": 2642, "url": "http://patches.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1580292549-27439-12-git-send-email-matan@mellanox.com/mbox/", "series": [ { "id": 8335, "url": "http://patches.dpdk.org/api/series/8335/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=8335", "date": "2020-01-29T10:08:56", "name": "Introduce mlx5 vDPA driver", "version": 2, "mbox": "http://patches.dpdk.org/series/8335/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/65298/comments/", "check": "fail", "checks": "http://patches.dpdk.org/api/patches/65298/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 73F65A0531;\n\tWed, 29 Jan 2020 11:11:28 +0100 (CET)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id EFF171BFB4;\n\tWed, 29 Jan 2020 11:10:25 +0100 (CET)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n by dpdk.org (Postfix) with ESMTP id 614B51BFB2\n for <dev@dpdk.org>; Wed, 29 Jan 2020 11:10:24 +0100 (CET)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n asafp@mellanox.com)\n with ESMTPS (AES256-SHA encrypted); 29 Jan 2020 12:10:23 +0200", "from pegasus07.mtr.labs.mlnx (pegasus07.mtr.labs.mlnx\n [10.210.16.112])\n by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 00TA9BHR032108;\n Wed, 29 Jan 2020 12:10:23 +0200" ], "From": "Matan Azrad <matan@mellanox.com>", "To": "dev@dpdk.org, Viacheslav Ovsiienko <viacheslavo@mellanox.com>", "Cc": "Maxime Coquelin <maxime.coquelin@redhat.com>", "Date": "Wed, 29 Jan 2020 10:09:07 +0000", "Message-Id": "<1580292549-27439-12-git-send-email-matan@mellanox.com>", "X-Mailer": "git-send-email 1.8.3.1", "In-Reply-To": "<1580292549-27439-1-git-send-email-matan@mellanox.com>", "References": "<1579539790-3882-1-git-send-email-matan@mellanox.com>\n <1580292549-27439-1-git-send-email-matan@mellanox.com>", "Subject": "[dpdk-dev] [PATCH v2 11/13] vdpa/mlx5: support live migration", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add support for live migration feature by the HW:\n\tCreate a single Mkey that maps the memory address space of the\n\t\tVHOST live migration log file.\n\tModify VIRTIO_NET_Q object and provide vhost_log_page,\n\t\tdirty_bitmap_mkey, dirty_bitmap_size, dirty_bitmap_addr\n\t\tand dirty_bitmap_dump_enable.\n\tModify VIRTIO_NET_Q object and move state to SUSPEND.\n\tQuery VIRTIO_NET_Q and get hw_available_idx and hw_used_idx.\n\nSigned-off-by: Matan Azrad <matan@mellanox.com>\nAcked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>\n---\n doc/guides/vdpadevs/features/mlx5.ini | 1 +\n drivers/vdpa/mlx5/Makefile | 1 +\n drivers/vdpa/mlx5/meson.build | 1 +\n drivers/vdpa/mlx5/mlx5_vdpa.c | 44 +++++++++++-\n drivers/vdpa/mlx5/mlx5_vdpa.h | 55 ++++++++++++++\n drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 130 ++++++++++++++++++++++++++++++++++\n drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 7 +-\n 7 files changed, 236 insertions(+), 3 deletions(-)\n create mode 100644 drivers/vdpa/mlx5/mlx5_vdpa_lm.c", "diff": "diff --git a/doc/guides/vdpadevs/features/mlx5.ini b/doc/guides/vdpadevs/features/mlx5.ini\nindex e4ee34b..1da9c1b 100644\n--- a/doc/guides/vdpadevs/features/mlx5.ini\n+++ b/doc/guides/vdpadevs/features/mlx5.ini\n@@ -9,6 +9,7 @@ guest csum = Y\n host tso4 = Y\n host tso6 = Y\n version 1 = Y\n+log all = Y\n any layout = Y\n guest announce = Y\n mq = Y\ndiff --git a/drivers/vdpa/mlx5/Makefile b/drivers/vdpa/mlx5/Makefile\nindex 2f70a98..4d1f528 100644\n--- a/drivers/vdpa/mlx5/Makefile\n+++ b/drivers/vdpa/mlx5/Makefile\n@@ -12,6 +12,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_VDPA_PMD) += mlx5_vdpa_mem.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_VDPA_PMD) += mlx5_vdpa_event.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_VDPA_PMD) += mlx5_vdpa_virtq.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_VDPA_PMD) += mlx5_vdpa_steer.c\n+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_VDPA_PMD) += mlx5_vdpa_lm.c\n \n \n # Basic CFLAGS.\ndiff --git a/drivers/vdpa/mlx5/meson.build b/drivers/vdpa/mlx5/meson.build\nindex 2849178..2e521b8 100644\n--- a/drivers/vdpa/mlx5/meson.build\n+++ b/drivers/vdpa/mlx5/meson.build\n@@ -16,6 +16,7 @@ sources = files(\n \t'mlx5_vdpa_event.c',\n \t'mlx5_vdpa_virtq.c',\n \t'mlx5_vdpa_steer.c',\n+\t'mlx5_vdpa_lm.c',\n )\n cflags_options = [\n \t'-std=c11',\ndiff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c\nindex 71189c4..4ce0ba0 100644\n--- a/drivers/vdpa/mlx5/mlx5_vdpa.c\n+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c\n@@ -19,7 +19,8 @@\n \t\t\t (1ULL << VIRTIO_F_ANY_LAYOUT) | \\\n \t\t\t (1ULL << VIRTIO_NET_F_MQ) | \\\n \t\t\t (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \\\n-\t\t\t (1ULL << VIRTIO_F_ORDER_PLATFORM))\n+\t\t\t (1ULL << VIRTIO_F_ORDER_PLATFORM) | \\\n+\t\t\t (1ULL << VHOST_F_LOG_ALL))\n \n #define MLX5_VDPA_PROTOCOL_FEATURES \\\n \t\t\t ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \\\n@@ -127,6 +128,45 @@\n \treturn mlx5_vdpa_virtq_enable(virtq, state);\n }\n \n+static int\n+mlx5_vdpa_features_set(int vid)\n+{\n+\tint did = rte_vhost_get_vdpa_device_id(vid);\n+\tstruct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);\n+\tuint64_t log_base, log_size;\n+\tuint64_t features;\n+\tint ret;\n+\n+\tif (priv == NULL) {\n+\t\tDRV_LOG(ERR, \"Invalid device id: %d.\", did);\n+\t\treturn -EINVAL;\n+\t}\n+\tret = rte_vhost_get_negotiated_features(vid, &features);\n+\tif (ret) {\n+\t\tDRV_LOG(ERR, \"Failed to get negotiated features.\");\n+\t\treturn ret;\n+\t}\n+\tif (RTE_VHOST_NEED_LOG(features)) {\n+\t\tret = rte_vhost_get_log_base(vid, &log_base, &log_size);\n+\t\tif (ret) {\n+\t\t\tDRV_LOG(ERR, \"Failed to get log base.\");\n+\t\t\treturn ret;\n+\t\t}\n+\t\tret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);\n+\t\tif (ret) {\n+\t\t\tDRV_LOG(ERR, \"Failed to set dirty bitmap.\");\n+\t\t\treturn ret;\n+\t\t}\n+\t\tDRV_LOG(INFO, \"mlx5 vdpa: enabling dirty logging...\");\n+\t\tret = mlx5_vdpa_logging_enable(priv, 1);\n+\t\tif (ret) {\n+\t\t\tDRV_LOG(ERR, \"Failed t enable dirty logging.\");\n+\t\t\treturn ret;\n+\t\t}\n+\t}\n+\treturn 0;\n+}\n+\n static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {\n \t.get_queue_num = mlx5_vdpa_get_queue_num,\n \t.get_features = mlx5_vdpa_get_vdpa_features,\n@@ -134,7 +174,7 @@\n \t.dev_conf = NULL,\n \t.dev_close = NULL,\n \t.set_vring_state = mlx5_vdpa_set_vring_state,\n-\t.set_features = NULL,\n+\t.set_features = mlx5_vdpa_features_set,\n \t.migration_done = NULL,\n \t.get_vfio_group_fd = NULL,\n \t.get_vfio_device_fd = NULL,\ndiff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h\nindex af78ea1..70264e4 100644\n--- a/drivers/vdpa/mlx5/mlx5_vdpa.h\n+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h\n@@ -244,4 +244,59 @@ int mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,\n */\n int mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv);\n \n+/**\n+ * Enable\\Disable live migration logging.\n+ *\n+ * @param[in] priv\n+ * The vdpa driver private structure.\n+ * @param[in] enable\n+ * Set for enable, unset for disable.\n+ *\n+ * @return\n+ * 0 on success, a negative value otherwise.\n+ */\n+int mlx5_vdpa_logging_enable(struct mlx5_vdpa_priv *priv, int enable);\n+\n+/**\n+ * Set dirty bitmap logging to allow live migration.\n+ *\n+ * @param[in] priv\n+ * The vdpa driver private structure.\n+ * @param[in] log_base\n+ * Vhost log base.\n+ * @param[in] log_size\n+ * Vhost log size.\n+ *\n+ * @return\n+ * 0 on success, a negative value otherwise.\n+ */\n+int mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,\n+\t\t\t uint64_t log_size);\n+\n+/**\n+ * Log all virtqs information for live migration.\n+ *\n+ * @param[in] priv\n+ * The vdpa driver private structure.\n+ * @param[in] enable\n+ * Set for enable, unset for disable.\n+ *\n+ * @return\n+ * 0 on success, a negative value otherwise.\n+ */\n+int mlx5_vdpa_lm_log(struct mlx5_vdpa_priv *priv);\n+\n+/**\n+ * Modify virtq state to be ready or suspend.\n+ *\n+ * @param[in] virtq\n+ * The vdpa driver private virtq structure.\n+ * @param[in] state\n+ * Set for ready, otherwise suspend.\n+ *\n+ * @return\n+ * 0 on success, a negative value otherwise.\n+ */\n+int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state);\n+\n #endif /* RTE_PMD_MLX5_VDPA_H_ */\ndiff --git a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c\nnew file mode 100644\nindex 0000000..cfeec5f\n--- /dev/null\n+++ b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c\n@@ -0,0 +1,130 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2019 Mellanox Technologies, Ltd\n+ */\n+#include <rte_malloc.h>\n+#include <rte_errno.h>\n+\n+#include \"mlx5_vdpa_utils.h\"\n+#include \"mlx5_vdpa.h\"\n+\n+\n+int\n+mlx5_vdpa_logging_enable(struct mlx5_vdpa_priv *priv, int enable)\n+{\n+\tstruct mlx5_devx_virtq_attr attr = {\n+\t\t.type = MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE,\n+\t\t.dirty_bitmap_dump_enable = enable,\n+\t};\n+\tstruct mlx5_vdpa_virtq *virtq;\n+\n+\tSLIST_FOREACH(virtq, &priv->virtq_list, next) {\n+\t\tattr.queue_index = virtq->index;\n+\t\tif (mlx5_devx_cmd_modify_virtq(virtq->virtq, &attr)) {\n+\t\t\tDRV_LOG(ERR, \"Failed to modify virtq %d logging.\",\n+\t\t\t\tvirtq->index);\n+\t\t\treturn -1;\n+\t\t}\n+\t}\n+\treturn 0;\n+}\n+\n+int\n+mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,\n+\t\t\t uint64_t log_size)\n+{\n+\tstruct mlx5_devx_mkey_attr mkey_attr = {\n+\t\t\t.addr = (uintptr_t)log_base,\n+\t\t\t.size = log_size,\n+\t\t\t.pd = priv->pdn,\n+\t\t\t.pg_access = 1,\n+\t\t\t.klm_array = NULL,\n+\t\t\t.klm_num = 0,\n+\t};\n+\tstruct mlx5_devx_virtq_attr attr = {\n+\t\t.type = MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS,\n+\t\t.dirty_bitmap_addr = log_base,\n+\t\t.dirty_bitmap_size = log_size,\n+\t};\n+\tstruct mlx5_vdpa_query_mr *mr = rte_malloc(__func__, sizeof(*mr), 0);\n+\tstruct mlx5_vdpa_virtq *virtq;\n+\n+\tif (!mr) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate mem for lm mr.\");\n+\t\treturn -1;\n+\t}\n+\tmr->umem = mlx5_glue->devx_umem_reg(priv->ctx,\n+\t\t\t\t\t (void *)(uintptr_t)log_base,\n+\t\t\t\t\t log_size, IBV_ACCESS_LOCAL_WRITE);\n+\tif (!mr->umem) {\n+\t\tDRV_LOG(ERR, \"Failed to register umem for lm mr.\");\n+\t\tgoto err;\n+\t}\n+\tmkey_attr.umem_id = mr->umem->umem_id;\n+\tmr->mkey = mlx5_devx_cmd_mkey_create(priv->ctx, &mkey_attr);\n+\tif (!mr->mkey) {\n+\t\tDRV_LOG(ERR, \"Failed to create Mkey for lm.\");\n+\t\tgoto err;\n+\t}\n+\tattr.dirty_bitmap_mkey = mr->mkey->id;\n+\tSLIST_FOREACH(virtq, &priv->virtq_list, next) {\n+\t\tattr.queue_index = virtq->index;\n+\t\tif (mlx5_devx_cmd_modify_virtq(virtq->virtq, &attr)) {\n+\t\t\tDRV_LOG(ERR, \"Failed to modify virtq %d for lm.\",\n+\t\t\t\tvirtq->index);\n+\t\t\tgoto err;\n+\t\t}\n+\t}\n+\tmr->is_indirect = 0;\n+\tSLIST_INSERT_HEAD(&priv->mr_list, mr, next);\n+\treturn 0;\n+err:\n+\tif (mr->mkey)\n+\t\tmlx5_devx_cmd_destroy(mr->mkey);\n+\tif (mr->umem)\n+\t\tmlx5_glue->devx_umem_dereg(mr->umem);\n+\trte_free(mr);\n+\treturn -1;\n+}\n+\n+#define MLX5_VDPA_USED_RING_LEN(size) \\\n+\t((size) * sizeof(struct vring_used_elem) + sizeof(uint16_t) * 3)\n+\n+int\n+mlx5_vdpa_lm_log(struct mlx5_vdpa_priv *priv)\n+{\n+\tstruct mlx5_devx_virtq_attr attr = {0};\n+\tstruct mlx5_vdpa_virtq *virtq;\n+\tuint64_t features;\n+\tint ret = rte_vhost_get_negotiated_features(priv->vid, &features);\n+\n+\tif (ret) {\n+\t\tDRV_LOG(ERR, \"Failed to get negotiated features.\");\n+\t\treturn -1;\n+\t}\n+\tif (RTE_VHOST_NEED_LOG(features)) {\n+\t\tSLIST_FOREACH(virtq, &priv->virtq_list, next) {\n+\t\t\tret = mlx5_vdpa_virtq_modify(virtq, 0);\n+\t\t\tif (ret)\n+\t\t\t\treturn -1;\n+\t\t\tif (mlx5_devx_cmd_query_virtq(virtq->virtq, &attr)) {\n+\t\t\t\tDRV_LOG(ERR, \"Failed to query virtq %d.\",\n+\t\t\t\t\tvirtq->index);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t\tDRV_LOG(INFO, \"Query vid %d vring %d: hw_available_idx=\"\n+\t\t\t\t\"%d, hw_used_index=%d\", priv->vid, virtq->index,\n+\t\t\t\tattr.hw_available_index, attr.hw_used_index);\n+\t\t\tret = rte_vhost_set_vring_base(priv->vid, virtq->index,\n+\t\t\t\t\t\t attr.hw_available_index,\n+\t\t\t\t\t\t attr.hw_used_index);\n+\t\t\tif (ret) {\n+\t\t\t\tDRV_LOG(ERR, \"Failed to set virtq %d base.\",\n+\t\t\t\t\tvirtq->index);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t\trte_vhost_log_used_vring(priv->vid, virtq->index, 0,\n+\t\t\t\t MLX5_VDPA_USED_RING_LEN(virtq->vq_size));\n+\t\t}\n+\t}\n+\treturn 0;\n+}\ndiff --git a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c\nindex 91347e9..af058a1 100644\n--- a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c\n+++ b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c\n@@ -112,7 +112,7 @@\n \tpriv->features = 0;\n }\n \n-static int\n+int\n mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state)\n {\n \tstruct mlx5_devx_virtq_attr attr = {\n@@ -253,6 +253,11 @@\n \tif (mlx5_vdpa_virtq_modify(virtq, 1))\n \t\tgoto error;\n \tvirtq->enable = 1;\n+\tvirtq->priv = priv;\n+\t/* Be sure notifications are not missed during configuration. */\n+\tclaim_zero(rte_vhost_enable_guest_notification(priv->vid, index, 1));\n+\trte_write32(virtq->index, priv->virtq_db_addr);\n+\t/* Setup doorbell mapping. */\n \tvirtq->intr_handle.fd = vq.kickfd;\n \tvirtq->intr_handle.type = RTE_INTR_HANDLE_EXT;\n \tif (rte_intr_callback_register(&virtq->intr_handle,\n", "prefixes": [ "v2", "11/13" ] }