Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/106687/?format=api
{ "id": 106687, "url": "http://patches.dpdk.org/api/patches/106687/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1643425417-215270-6-git-send-email-andy.pei@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1643425417-215270-6-git-send-email-andy.pei@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1643425417-215270-6-git-send-email-andy.pei@intel.com", "date": "2022-01-29T03:03:27", "name": "[v3,05/15] vdpa/ifc: add blk dev sw live migration", "commit_ref": null, "pull_url": null, "state": "changes-requested", "archived": true, "hash": "5287279d6207bc8fba9973f367627c327d10fe42", "submitter": { "id": 1185, "url": "http://patches.dpdk.org/api/people/1185/?format=api", "name": "Pei, Andy", "email": "andy.pei@intel.com" }, "delegate": { "id": 2642, "url": "http://patches.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1643425417-215270-6-git-send-email-andy.pei@intel.com/mbox/", "series": [ { "id": 21420, "url": "http://patches.dpdk.org/api/series/21420/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=21420", "date": "2022-01-29T03:03:22", "name": "add virtio_blk device support to vdpa/ifc", "version": 3, "mbox": "http://patches.dpdk.org/series/21420/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/106687/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/106687/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 729D8A00C4;\n\tSat, 29 Jan 2022 04:10:00 +0100 (CET)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 0600D427A7;\n\tSat, 29 Jan 2022 04:09:51 +0100 (CET)", "from mga04.intel.com (mga04.intel.com [192.55.52.120])\n by mails.dpdk.org (Postfix) with ESMTP id 2405D4279D\n for <dev@dpdk.org>; Sat, 29 Jan 2022 04:09:48 +0100 (CET)", "from fmsmga008.fm.intel.com ([10.253.24.58])\n by fmsmga104.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 28 Jan 2022 19:09:48 -0800", "from dpdk-dipei.sh.intel.com ([10.67.111.91])\n by fmsmga008.fm.intel.com with ESMTP; 28 Jan 2022 19:09:42 -0800" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1643425789; x=1674961789;\n h=from:to:cc:subject:date:message-id:in-reply-to: references;\n bh=mmQdtEyWueR0IpOBtyGW8/NFuPKxBfvpQaweaOKnDns=;\n b=Q13V6agXL3DCv2ni38tZ2CeWvdRE31eMiJnXCGUl195dJDDt5unX/BKY\n Lut+JL0MApqqxTjyNpP5LlnEgrbKJxxpxlVu1Ve/ZKeQMkpBLK7FdvSyX\n ndB6pgXmEsVOibEZf1l2rQFe9rGGoJ1dGlBpJoOTNQhMH9vAzZ9r2mu3Y\n 4fE4dw3Dh1fOYUa/zR3ztcU7RCBRilL2qHJwLsRfBl4/0NUvAIwdzjJcF\n MAT7YAAcvr0X3g+1v+cnm35UKSnXex3h3cI9wUCEUMSLbWv8qxujIupLl\n GGf6OcedZqXlEtgto7A54YC/rbdzLUSyq/JicQcYn/opUCw3km3jeQDH/ g==;", "X-IronPort-AV": [ "E=McAfee;i=\"6200,9189,10241\"; a=\"246076619\"", "E=Sophos;i=\"5.88,325,1635231600\"; d=\"scan'208\";a=\"246076619\"", "E=Sophos;i=\"5.88,325,1635231600\"; d=\"scan'208\";a=\"582047799\"" ], "X-ExtLoop1": "1", "From": "Andy Pei <andy.pei@intel.com>", "To": "dev@dpdk.org", "Cc": "chenbo.xia@intel.com, maxime.coquelin@redhat.com, gang.cao@intel.com,\n changpeng.liu@intel.com", "Subject": "[PATCH v3 05/15] vdpa/ifc: add blk dev sw live migration", "Date": "Sat, 29 Jan 2022 11:03:27 +0800", "Message-Id": "<1643425417-215270-6-git-send-email-andy.pei@intel.com>", "X-Mailer": "git-send-email 1.8.3.1", "In-Reply-To": "<1643425417-215270-1-git-send-email-andy.pei@intel.com>", "References": "<1643093258-47258-2-git-send-email-andy.pei@intel.com>\n <1643425417-215270-1-git-send-email-andy.pei@intel.com>", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Enable virtio blk sw live migration relay callfd and log the dirty page.\nIn this version we ignore the write cmd and still mark it dirty.\n\nSigned-off-by: Andy Pei <andy.pei@intel.com>\n---\n drivers/vdpa/ifc/base/ifcvf.c | 4 +-\n drivers/vdpa/ifc/base/ifcvf.h | 6 ++\n drivers/vdpa/ifc/ifcvf_vdpa.c | 128 +++++++++++++++++++++++++++++++++++-------\n 3 files changed, 116 insertions(+), 22 deletions(-)", "diff": "diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c\nindex 721cb1d..3a69e53 100644\n--- a/drivers/vdpa/ifc/base/ifcvf.c\n+++ b/drivers/vdpa/ifc/base/ifcvf.c\n@@ -189,7 +189,7 @@\n \tIFCVF_WRITE_REG32(val >> 32, hi);\n }\n \n-STATIC int\n+int\n ifcvf_hw_enable(struct ifcvf_hw *hw)\n {\n \tstruct ifcvf_pci_common_cfg *cfg;\n@@ -238,7 +238,7 @@\n \treturn 0;\n }\n \n-STATIC void\n+void\n ifcvf_hw_disable(struct ifcvf_hw *hw)\n {\n \tu32 i;\ndiff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h\nindex 769c603..6dd7925 100644\n--- a/drivers/vdpa/ifc/base/ifcvf.h\n+++ b/drivers/vdpa/ifc/base/ifcvf.h\n@@ -179,4 +179,10 @@ struct ifcvf_hw {\n u64\n ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid);\n \n+int\n+ifcvf_hw_enable(struct ifcvf_hw *hw);\n+\n+void\n+ifcvf_hw_disable(struct ifcvf_hw *hw);\n+\n #endif /* _IFCVF_H_ */\ndiff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c\nindex 4f99bb3..a930825 100644\n--- a/drivers/vdpa/ifc/ifcvf_vdpa.c\n+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c\n@@ -332,10 +332,67 @@ struct rte_vdpa_dev_info {\n \n \trte_vhost_get_negotiated_features(vid, &features);\n \tif (RTE_VHOST_NEED_LOG(features)) {\n-\t\tifcvf_disable_logging(hw);\n-\t\trte_vhost_get_log_base(internal->vid, &log_base, &log_size);\n-\t\trte_vfio_container_dma_unmap(internal->vfio_container_fd,\n-\t\t\t\tlog_base, IFCVF_LOG_BASE, log_size);\n+\t\tif (internal->device_type == IFCVF_NET) {\n+\t\t\tifcvf_disable_logging(hw);\n+\t\t\trte_vhost_get_log_base(internal->vid, &log_base,\n+\t\t\t\t&log_size);\n+\t\t\trte_vfio_container_dma_unmap(\n+\t\t\t\tinternal->vfio_container_fd, log_base,\n+\t\t\t\tIFCVF_LOG_BASE, log_size);\n+\t\t}\n+\t\t/* IFCVF marks dirty memory pages for only packet buffer,\n+\t\t * SW helps to mark the used ring as dirty after device stops.\n+\t\t */\n+\t\tfor (i = 0; i < hw->nr_vring; i++) {\n+\t\t\tlen = IFCVF_USED_RING_LEN(hw->vring[i].size);\n+\t\t\trte_vhost_log_used_vring(vid, i, 0, len);\n+\t\t}\n+\t}\n+}\n+\n+static void\n+vdpa_ifcvf_blk_pause(struct ifcvf_internal *internal)\n+{\n+\tstruct ifcvf_hw *hw = &internal->hw;\n+\tstruct rte_vhost_vring vq;\n+\tint i, vid;\n+\tuint64_t features = 0;\n+\tuint64_t log_base = 0, log_size = 0;\n+\tuint64_t len;\n+\n+\tvid = internal->vid;\n+\n+\tif (internal->device_type == IFCVF_BLK) {\n+\t\tfor (i = 0; i < hw->nr_vring; i++) {\n+\t\t\trte_vhost_get_vhost_vring(internal->vid, i, &vq);\n+\t\t\twhile (vq.avail->idx != vq.used->idx) {\n+\t\t\t\tifcvf_notify_queue(hw, i);\n+\t\t\t\tusleep(10);\n+\t\t\t}\n+\t\t\thw->vring[i].last_avail_idx = vq.avail->idx;\n+\t\t\thw->vring[i].last_used_idx = vq.used->idx;\n+\t\t}\n+\t}\n+\n+\tifcvf_hw_disable(hw);\n+\n+\tfor (i = 0; i < hw->nr_vring; i++)\n+\t\trte_vhost_set_vring_base(vid, i, hw->vring[i].last_avail_idx,\n+\t\t\t\thw->vring[i].last_used_idx);\n+\n+\tif (internal->sw_lm)\n+\t\treturn;\n+\n+\trte_vhost_get_negotiated_features(vid, &features);\n+\tif (RTE_VHOST_NEED_LOG(features)) {\n+\t\tif (internal->device_type == IFCVF_NET) {\n+\t\t\tifcvf_disable_logging(hw);\n+\t\t\trte_vhost_get_log_base(internal->vid, &log_base,\n+\t\t\t\t&log_size);\n+\t\t\trte_vfio_container_dma_unmap(\n+\t\t\t\tinternal->vfio_container_fd, log_base,\n+\t\t\t\tIFCVF_LOG_BASE, log_size);\n+\t\t}\n \t\t/*\n \t\t * IFCVF marks dirty memory pages for only packet buffer,\n \t\t * SW helps to mark the used ring as dirty after device stops.\n@@ -661,15 +718,17 @@ struct rte_vdpa_dev_info {\n \t\t}\n \t\thw->vring[i].avail = gpa;\n \n-\t\t/* Direct I/O for Tx queue, relay for Rx queue */\n-\t\tif (i & 1) {\n+\t\t/* NETWORK: Direct I/O for Tx queue, relay for Rx queue\n+\t\t * BLK: relay every queue\n+\t\t */\n+\t\tif ((i & 1) && (internal->device_type == IFCVF_NET)) {\n \t\t\tgpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);\n \t\t\tif (gpa == 0) {\n \t\t\t\tDRV_LOG(ERR, \"Fail to get GPA for used ring.\");\n \t\t\t\treturn -1;\n \t\t\t}\n \t\t\thw->vring[i].used = gpa;\n-\t\t} else {\n+\t\t} else if (internal->device_type == IFCVF_BLK) {\n \t\t\thw->vring[i].used = m_vring_iova +\n \t\t\t\t(char *)internal->m_vring[i].used -\n \t\t\t\t(char *)internal->m_vring[i].desc;\n@@ -688,7 +747,10 @@ struct rte_vdpa_dev_info {\n \t}\n \thw->nr_vring = nr_vring;\n \n-\treturn ifcvf_start_hw(&internal->hw);\n+\tif (internal->device_type == IFCVF_NET)\n+\t\treturn ifcvf_start_hw(&internal->hw);\n+\telse if (internal->device_type == IFCVF_BLK)\n+\t\treturn ifcvf_hw_enable(&internal->hw);\n \n error:\n \tfor (i = 0; i < nr_vring; i++)\n@@ -713,8 +775,10 @@ struct rte_vdpa_dev_info {\n \n \tfor (i = 0; i < hw->nr_vring; i++) {\n \t\t/* synchronize remaining new used entries if any */\n-\t\tif ((i & 1) == 0)\n+\t\tif (((i & 1) == 0 && internal->device_type == IFCVF_NET) ||\n+\t\t internal->device_type == IFCVF_BLK) {\n \t\t\tupdate_used_ring(internal, i);\n+\t\t}\n \n \t\trte_vhost_get_vhost_vring(vid, i, &vq);\n \t\tlen = IFCVF_USED_RING_LEN(vq.size);\n@@ -726,6 +790,8 @@ struct rte_vdpa_dev_info {\n \t\t\t(uint64_t)(uintptr_t)internal->m_vring[i].desc,\n \t\t\tm_vring_iova, size);\n \n+\t\thw->vring[i].last_avail_idx = vq.used->idx;\n+\t\thw->vring[i].last_used_idx = vq.used->idx;\n \t\trte_vhost_set_vring_base(vid, i, hw->vring[i].last_avail_idx,\n \t\t\t\thw->vring[i].last_used_idx);\n \t\trte_free(internal->m_vring[i].desc);\n@@ -776,17 +842,36 @@ struct rte_vdpa_dev_info {\n \t\t}\n \t}\n \n-\tfor (qid = 0; qid < q_num; qid += 2) {\n-\t\tev.events = EPOLLIN | EPOLLPRI;\n-\t\t/* leave a flag to mark it's for interrupt */\n-\t\tev.data.u64 = 1 | qid << 1 |\n-\t\t\t(uint64_t)internal->intr_fd[qid] << 32;\n-\t\tif (epoll_ctl(epfd, EPOLL_CTL_ADD, internal->intr_fd[qid], &ev)\n-\t\t\t\t< 0) {\n-\t\t\tDRV_LOG(ERR, \"epoll add error: %s\", strerror(errno));\n-\t\t\treturn NULL;\n+\tif (internal->device_type == IFCVF_NET) {\n+\t\tfor (qid = 0; qid < q_num; qid += 2) {\n+\t\t\tev.events = EPOLLIN | EPOLLPRI;\n+\t\t\t/* leave a flag to mark it's for interrupt */\n+\t\t\tev.data.u64 = 1 | qid << 1 |\n+\t\t\t\t(uint64_t)internal->intr_fd[qid] << 32;\n+\t\t\tif (epoll_ctl(epfd, EPOLL_CTL_ADD,\n+\t\t\t\t internal->intr_fd[qid], &ev)\n+\t\t\t\t\t< 0) {\n+\t\t\t\tDRV_LOG(ERR, \"epoll add error: %s\",\n+\t\t\t\t\tstrerror(errno));\n+\t\t\t\treturn NULL;\n+\t\t\t}\n+\t\t\tupdate_used_ring(internal, qid);\n+\t\t}\n+\t} else if (internal->device_type == IFCVF_BLK) {\n+\t\tfor (qid = 0; qid < q_num; qid += 1) {\n+\t\t\tev.events = EPOLLIN | EPOLLPRI;\n+\t\t\t/* leave a flag to mark it's for interrupt */\n+\t\t\tev.data.u64 = 1 | qid << 1 |\n+\t\t\t\t(uint64_t)internal->intr_fd[qid] << 32;\n+\t\t\tif (epoll_ctl(epfd, EPOLL_CTL_ADD,\n+\t\t\t\t internal->intr_fd[qid], &ev)\n+\t\t\t\t\t< 0) {\n+\t\t\t\tDRV_LOG(ERR, \"epoll add error: %s\",\n+\t\t\t\t\tstrerror(errno));\n+\t\t\t\treturn NULL;\n+\t\t\t}\n+\t\t\tupdate_used_ring(internal, qid);\n \t\t}\n-\t\tupdate_used_ring(internal, qid);\n \t}\n \n \t/* start relay with a first kick */\n@@ -874,7 +959,10 @@ struct rte_vdpa_dev_info {\n \n \t/* stop the direct IO data path */\n \tunset_notify_relay(internal);\n-\tvdpa_ifcvf_stop(internal);\n+\tif (internal->device_type == IFCVF_NET)\n+\t\tvdpa_ifcvf_stop(internal);\n+\telse if (internal->device_type == IFCVF_BLK)\n+\t\tvdpa_ifcvf_blk_pause(internal);\n \tvdpa_disable_vfio_intr(internal);\n \n \tret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false);\n", "prefixes": [ "v3", "05/15" ] }