[v3,05/11] vdpa/nfp: add the live migration logic
Checks
Commit Message
From: Xinying Yu <xinying.yu@corigine.com>
Add the basic logic of software live migration.
Unset the ring notify area to stop the direct IO datapath if the
device support, then we can setup the vring relay to help the
live migration.
Signed-off-by: Xinying Yu <xinying.yu@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
---
drivers/vdpa/nfp/nfp_vdpa.c | 66 +++++++++++++++++++++++++++++++-
drivers/vdpa/nfp/nfp_vdpa_core.c | 2 +
drivers/vdpa/nfp/nfp_vdpa_core.h | 4 ++
3 files changed, 70 insertions(+), 2 deletions(-)
Comments
On 6/17/24 08:27, Chaoyong He wrote:
> From: Xinying Yu <xinying.yu@corigine.com>
>
> Add the basic logic of software live migration.
>
> Unset the ring notify area to stop the direct IO datapath if the
> device support, then we can setup the vring relay to help the
> live migration.
>
> Signed-off-by: Xinying Yu <xinying.yu@corigine.com>
> Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
> Reviewed-by: Long Wu <long.wu@corigine.com>
> Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
> ---
> drivers/vdpa/nfp/nfp_vdpa.c | 66 +++++++++++++++++++++++++++++++-
> drivers/vdpa/nfp/nfp_vdpa_core.c | 2 +
> drivers/vdpa/nfp/nfp_vdpa_core.h | 4 ++
> 3 files changed, 70 insertions(+), 2 deletions(-)
>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
Thanks,
Maxime
@@ -603,6 +603,30 @@ update_datapath(struct nfp_vdpa_dev *device)
return ret;
}
+static int
+nfp_vdpa_sw_fallback(struct nfp_vdpa_dev *device)
+{
+ int ret;
+ int vid = device->vid;
+
+ /* Stop the direct IO data path */
+ nfp_vdpa_unset_notify_relay(device);
+ nfp_vdpa_disable_vfio_intr(device);
+
+ ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false);
+ if ((ret != 0) && (ret != -ENOTSUP)) {
+ DRV_VDPA_LOG(ERR, "Unset the host notifier failed.");
+ goto error;
+ }
+
+ device->hw.sw_fallback_running = true;
+
+ return 0;
+
+error:
+ return ret;
+}
+
static int
nfp_vdpa_dev_config(int vid)
{
@@ -646,8 +670,18 @@ nfp_vdpa_dev_close(int vid)
}
device = node->device;
- rte_atomic_store_explicit(&device->dev_attached, 0, rte_memory_order_relaxed);
- update_datapath(device);
+ if (device->hw.sw_fallback_running) {
+ device->hw.sw_fallback_running = false;
+
+ rte_atomic_store_explicit(&device->dev_attached, 0,
+ rte_memory_order_relaxed);
+ rte_atomic_store_explicit(&device->running, 0,
+ rte_memory_order_relaxed);
+ } else {
+ rte_atomic_store_explicit(&device->dev_attached, 0,
+ rte_memory_order_relaxed);
+ update_datapath(device);
+ }
return 0;
}
@@ -770,7 +804,35 @@ nfp_vdpa_get_protocol_features(struct rte_vdpa_device *vdev __rte_unused,
static int
nfp_vdpa_set_features(int32_t vid)
{
+ int ret;
+ uint64_t features = 0;
+ struct nfp_vdpa_dev *device;
+ struct rte_vdpa_device *vdev;
+ struct nfp_vdpa_dev_node *node;
+
DRV_VDPA_LOG(DEBUG, "Start vid=%d", vid);
+
+ vdev = rte_vhost_get_vdpa_device(vid);
+ node = nfp_vdpa_find_node_by_vdev(vdev);
+ if (node == NULL) {
+ DRV_VDPA_LOG(ERR, "Invalid vDPA device: %p", vdev);
+ return -ENODEV;
+ }
+
+ rte_vhost_get_negotiated_features(vid, &features);
+
+ if (RTE_VHOST_NEED_LOG(features) == 0)
+ return 0;
+
+ device = node->device;
+ if (device->hw.sw_lm) {
+ ret = nfp_vdpa_sw_fallback(device);
+ if (ret != 0) {
+ DRV_VDPA_LOG(ERR, "Software fallback start failed");
+ return -1;
+ }
+ }
+
return 0;
}
@@ -91,6 +91,8 @@ nfp_vdpa_hw_init(struct nfp_vdpa_hw *vdpa_hw,
tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + tx_bar_off;
hw->qcp_cfg = tx_bar + NFP_QCP_QUEUE_ADDR_SZ;
+ vdpa_hw->sw_lm = true;
+
vdpa_hw->features = (1ULL << VIRTIO_F_VERSION_1) |
(1ULL << VIRTIO_F_IN_ORDER) |
(1ULL << VHOST_USER_F_PROTOCOL_FEATURES);
@@ -36,6 +36,10 @@ struct nfp_vdpa_hw {
uint8_t mac_addr[RTE_ETHER_ADDR_LEN];
uint8_t notify_region;
uint8_t nr_vring;
+
+ /** Software Live Migration */
+ bool sw_lm;
+ bool sw_fallback_running;
};
int nfp_vdpa_hw_init(struct nfp_vdpa_hw *vdpa_hw, struct rte_pci_device *dev);