[v3,18/19] vhost: add flag to enable postcopy live-migration
Checks
Commit Message
Postcopy live-migration feature require the application to
not populate the guest memory. As the vhost library cannot
prevent the application to that (e.g. preventing the
application to call mlockall()), the feature is disabled by
default.
The application should only enable the feature if it does not
force the guest memory to be populated.
In case the user passes the RTE_VHOST_USER_POSTCOPY_SUPPORT
flag at registration but the feature was not compiled,
registration fails.
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
doc/guides/prog_guide/vhost_lib.rst | 8 ++++++++
lib/librte_vhost/rte_vhost.h | 1 +
lib/librte_vhost/socket.c | 19 +++++++++++++++++--
3 files changed, 26 insertions(+), 2 deletions(-)
Comments
I think it'll be nice to have a warning in case zero-copy
and postcopy enabled at the same time. This warning will
clarify for users why postcopy is not enabled.
What do you think?
Maybe we can also move the check from the previous patch to
rte_vhost_driver_register() like it done for VIRTIO_F_IN_ORDER ?
We can't change the set of enabled features in runtime anyway.
Best regards, Ilya Maximets.
On 04.10.2018 11:14, Maxime Coquelin wrote:
> Postcopy live-migration feature require the application to
> not populate the guest memory. As the vhost library cannot
> prevent the application to that (e.g. preventing the
> application to call mlockall()), the feature is disabled by
> default.
>
> The application should only enable the feature if it does not
> force the guest memory to be populated.
>
> In case the user passes the RTE_VHOST_USER_POSTCOPY_SUPPORT
> flag at registration but the feature was not compiled,
> registration fails.
>
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
> doc/guides/prog_guide/vhost_lib.rst | 8 ++++++++
> lib/librte_vhost/rte_vhost.h | 1 +
> lib/librte_vhost/socket.c | 19 +++++++++++++++++--
> 3 files changed, 26 insertions(+), 2 deletions(-)
>
> diff --git a/doc/guides/prog_guide/vhost_lib.rst b/doc/guides/prog_guide/vhost_lib.rst
> index 77af4d775..c77df338f 100644
> --- a/doc/guides/prog_guide/vhost_lib.rst
> +++ b/doc/guides/prog_guide/vhost_lib.rst
> @@ -106,6 +106,14 @@ The following is an overview of some key Vhost API functions:
> Enabling this flag with these Qemu version results in Qemu being blocked
> when multiple queue pairs are declared.
>
> + - ``RTE_VHOST_USER_POSTCOPY_SUPPORT``
> +
> + Postcopy live-migration support will be enabled when this flag is set.
> + It is disabled by default.
> +
> + Enabling this flag should only be done when the calling application does
> + not pre-fault the guest shared memory, otherwise migration would fail.
> +
> * ``rte_vhost_driver_set_features(path, features)``
>
> This function sets the feature bits the vhost-user driver supports. The
> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
> index b3cc6990d..b26afbffa 100644
> --- a/lib/librte_vhost/rte_vhost.h
> +++ b/lib/librte_vhost/rte_vhost.h
> @@ -28,6 +28,7 @@ extern "C" {
> #define RTE_VHOST_USER_NO_RECONNECT (1ULL << 1)
> #define RTE_VHOST_USER_DEQUEUE_ZERO_COPY (1ULL << 2)
> #define RTE_VHOST_USER_IOMMU_SUPPORT (1ULL << 3)
> +#define RTE_VHOST_USER_POSTCOPY_SUPPORT (1ULL << 4)
>
> /** Protocol features. */
> #ifndef VHOST_USER_PROTOCOL_F_MQ
> diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
> index c04d3d305..3df303be8 100644
> --- a/lib/librte_vhost/socket.c
> +++ b/lib/librte_vhost/socket.c
> @@ -51,6 +51,8 @@ struct vhost_user_socket {
> uint64_t supported_features;
> uint64_t features;
>
> + uint64_t protocol_features;
> +
> /*
> * Device id to identify a specific backend device.
> * It's set to -1 for the default software implementation.
> @@ -731,7 +733,7 @@ rte_vhost_driver_get_protocol_features(const char *path,
> did = vsocket->vdpa_dev_id;
> vdpa_dev = rte_vdpa_get_device(did);
> if (!vdpa_dev || !vdpa_dev->ops->get_protocol_features) {
> - *protocol_features = VHOST_USER_PROTOCOL_FEATURES;
> + *protocol_features = vsocket->protocol_features;
> goto unlock_exit;
> }
>
> @@ -744,7 +746,7 @@ rte_vhost_driver_get_protocol_features(const char *path,
> goto unlock_exit;
> }
>
> - *protocol_features = VHOST_USER_PROTOCOL_FEATURES
> + *protocol_features = vsocket->protocol_features
> & vdpa_protocol_features;
>
> unlock_exit:
> @@ -863,6 +865,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
> vsocket->use_builtin_virtio_net = true;
> vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES;
> vsocket->features = VIRTIO_NET_SUPPORTED_FEATURES;
> + vsocket->protocol_features = VHOST_USER_PROTOCOL_FEATURES;
>
> /* Dequeue zero copy can't assure descriptors returned in order */
> if (vsocket->dequeue_zero_copy) {
> @@ -875,6 +878,18 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
> vsocket->features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
> }
>
> + if (!(flags & RTE_VHOST_USER_POSTCOPY_SUPPORT)) {
> + vsocket->protocol_features &=
> + ~(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT);
> + } else {
> +#ifndef RTE_LIBRTE_VHOST_POSTCOPY
> + RTE_LOG(ERR, VHOST_CONFIG,
> + "Postcopy requested but not compiled\n");
> + ret = -1;
> + goto out_mutex;
> +#endif
> + }
> +
> if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
> vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
> if (vsocket->reconnect && reconn_tid == 0) {
>
Have you missed this mail while preparing v4?
Best regards, Ilya Maximets.
On 05.10.2018 17:24, Ilya Maximets wrote:
> I think it'll be nice to have a warning in case zero-copy
> and postcopy enabled at the same time. This warning will
> clarify for users why postcopy is not enabled.
> What do you think?
>
> Maybe we can also move the check from the previous patch to
> rte_vhost_driver_register() like it done for VIRTIO_F_IN_ORDER ?
> We can't change the set of enabled features in runtime anyway.
>
> Best regards, Ilya Maximets.
>
> On 04.10.2018 11:14, Maxime Coquelin wrote:
>> Postcopy live-migration feature require the application to
>> not populate the guest memory. As the vhost library cannot
>> prevent the application to that (e.g. preventing the
>> application to call mlockall()), the feature is disabled by
>> default.
>>
>> The application should only enable the feature if it does not
>> force the guest memory to be populated.
>>
>> In case the user passes the RTE_VHOST_USER_POSTCOPY_SUPPORT
>> flag at registration but the feature was not compiled,
>> registration fails.
>>
>> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>> ---
>> doc/guides/prog_guide/vhost_lib.rst | 8 ++++++++
>> lib/librte_vhost/rte_vhost.h | 1 +
>> lib/librte_vhost/socket.c | 19 +++++++++++++++++--
>> 3 files changed, 26 insertions(+), 2 deletions(-)
>>
>> diff --git a/doc/guides/prog_guide/vhost_lib.rst b/doc/guides/prog_guide/vhost_lib.rst
>> index 77af4d775..c77df338f 100644
>> --- a/doc/guides/prog_guide/vhost_lib.rst
>> +++ b/doc/guides/prog_guide/vhost_lib.rst
>> @@ -106,6 +106,14 @@ The following is an overview of some key Vhost API functions:
>> Enabling this flag with these Qemu version results in Qemu being blocked
>> when multiple queue pairs are declared.
>>
>> + - ``RTE_VHOST_USER_POSTCOPY_SUPPORT``
>> +
>> + Postcopy live-migration support will be enabled when this flag is set.
>> + It is disabled by default.
>> +
>> + Enabling this flag should only be done when the calling application does
>> + not pre-fault the guest shared memory, otherwise migration would fail.
>> +
>> * ``rte_vhost_driver_set_features(path, features)``
>>
>> This function sets the feature bits the vhost-user driver supports. The
>> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
>> index b3cc6990d..b26afbffa 100644
>> --- a/lib/librte_vhost/rte_vhost.h
>> +++ b/lib/librte_vhost/rte_vhost.h
>> @@ -28,6 +28,7 @@ extern "C" {
>> #define RTE_VHOST_USER_NO_RECONNECT (1ULL << 1)
>> #define RTE_VHOST_USER_DEQUEUE_ZERO_COPY (1ULL << 2)
>> #define RTE_VHOST_USER_IOMMU_SUPPORT (1ULL << 3)
>> +#define RTE_VHOST_USER_POSTCOPY_SUPPORT (1ULL << 4)
>>
>> /** Protocol features. */
>> #ifndef VHOST_USER_PROTOCOL_F_MQ
>> diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
>> index c04d3d305..3df303be8 100644
>> --- a/lib/librte_vhost/socket.c
>> +++ b/lib/librte_vhost/socket.c
>> @@ -51,6 +51,8 @@ struct vhost_user_socket {
>> uint64_t supported_features;
>> uint64_t features;
>>
>> + uint64_t protocol_features;
>> +
>> /*
>> * Device id to identify a specific backend device.
>> * It's set to -1 for the default software implementation.
>> @@ -731,7 +733,7 @@ rte_vhost_driver_get_protocol_features(const char *path,
>> did = vsocket->vdpa_dev_id;
>> vdpa_dev = rte_vdpa_get_device(did);
>> if (!vdpa_dev || !vdpa_dev->ops->get_protocol_features) {
>> - *protocol_features = VHOST_USER_PROTOCOL_FEATURES;
>> + *protocol_features = vsocket->protocol_features;
>> goto unlock_exit;
>> }
>>
>> @@ -744,7 +746,7 @@ rte_vhost_driver_get_protocol_features(const char *path,
>> goto unlock_exit;
>> }
>>
>> - *protocol_features = VHOST_USER_PROTOCOL_FEATURES
>> + *protocol_features = vsocket->protocol_features
>> & vdpa_protocol_features;
>>
>> unlock_exit:
>> @@ -863,6 +865,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
>> vsocket->use_builtin_virtio_net = true;
>> vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES;
>> vsocket->features = VIRTIO_NET_SUPPORTED_FEATURES;
>> + vsocket->protocol_features = VHOST_USER_PROTOCOL_FEATURES;
>>
>> /* Dequeue zero copy can't assure descriptors returned in order */
>> if (vsocket->dequeue_zero_copy) {
>> @@ -875,6 +878,18 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
>> vsocket->features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
>> }
>>
>> + if (!(flags & RTE_VHOST_USER_POSTCOPY_SUPPORT)) {
>> + vsocket->protocol_features &=
>> + ~(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT);
>> + } else {
>> +#ifndef RTE_LIBRTE_VHOST_POSTCOPY
>> + RTE_LOG(ERR, VHOST_CONFIG,
>> + "Postcopy requested but not compiled\n");
>> + ret = -1;
>> + goto out_mutex;
>> +#endif
>> + }
>> +
>> if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
>> vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
>> if (vsocket->reconnect && reconn_tid == 0) {
>>
>
>
On 10/08/2018 06:25 PM, Ilya Maximets wrote:
> Have you missed this mail while preparing v4?
Yes I did... Thanks for the heads-up!
Maxime
> Best regards, Ilya Maximets.
>
> On 05.10.2018 17:24, Ilya Maximets wrote:
>> I think it'll be nice to have a warning in case zero-copy
>> and postcopy enabled at the same time. This warning will
>> clarify for users why postcopy is not enabled.
>> What do you think?
>>
>> Maybe we can also move the check from the previous patch to
>> rte_vhost_driver_register() like it done for VIRTIO_F_IN_ORDER ?
>> We can't change the set of enabled features in runtime anyway.
>>
>> Best regards, Ilya Maximets.
>>
>> On 04.10.2018 11:14, Maxime Coquelin wrote:
>>> Postcopy live-migration feature require the application to
>>> not populate the guest memory. As the vhost library cannot
>>> prevent the application to that (e.g. preventing the
>>> application to call mlockall()), the feature is disabled by
>>> default.
>>>
>>> The application should only enable the feature if it does not
>>> force the guest memory to be populated.
>>>
>>> In case the user passes the RTE_VHOST_USER_POSTCOPY_SUPPORT
>>> flag at registration but the feature was not compiled,
>>> registration fails.
>>>
>>> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>>> ---
>>> doc/guides/prog_guide/vhost_lib.rst | 8 ++++++++
>>> lib/librte_vhost/rte_vhost.h | 1 +
>>> lib/librte_vhost/socket.c | 19 +++++++++++++++++--
>>> 3 files changed, 26 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/doc/guides/prog_guide/vhost_lib.rst b/doc/guides/prog_guide/vhost_lib.rst
>>> index 77af4d775..c77df338f 100644
>>> --- a/doc/guides/prog_guide/vhost_lib.rst
>>> +++ b/doc/guides/prog_guide/vhost_lib.rst
>>> @@ -106,6 +106,14 @@ The following is an overview of some key Vhost API functions:
>>> Enabling this flag with these Qemu version results in Qemu being blocked
>>> when multiple queue pairs are declared.
>>>
>>> + - ``RTE_VHOST_USER_POSTCOPY_SUPPORT``
>>> +
>>> + Postcopy live-migration support will be enabled when this flag is set.
>>> + It is disabled by default.
>>> +
>>> + Enabling this flag should only be done when the calling application does
>>> + not pre-fault the guest shared memory, otherwise migration would fail.
>>> +
>>> * ``rte_vhost_driver_set_features(path, features)``
>>>
>>> This function sets the feature bits the vhost-user driver supports. The
>>> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
>>> index b3cc6990d..b26afbffa 100644
>>> --- a/lib/librte_vhost/rte_vhost.h
>>> +++ b/lib/librte_vhost/rte_vhost.h
>>> @@ -28,6 +28,7 @@ extern "C" {
>>> #define RTE_VHOST_USER_NO_RECONNECT (1ULL << 1)
>>> #define RTE_VHOST_USER_DEQUEUE_ZERO_COPY (1ULL << 2)
>>> #define RTE_VHOST_USER_IOMMU_SUPPORT (1ULL << 3)
>>> +#define RTE_VHOST_USER_POSTCOPY_SUPPORT (1ULL << 4)
>>>
>>> /** Protocol features. */
>>> #ifndef VHOST_USER_PROTOCOL_F_MQ
>>> diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
>>> index c04d3d305..3df303be8 100644
>>> --- a/lib/librte_vhost/socket.c
>>> +++ b/lib/librte_vhost/socket.c
>>> @@ -51,6 +51,8 @@ struct vhost_user_socket {
>>> uint64_t supported_features;
>>> uint64_t features;
>>>
>>> + uint64_t protocol_features;
>>> +
>>> /*
>>> * Device id to identify a specific backend device.
>>> * It's set to -1 for the default software implementation.
>>> @@ -731,7 +733,7 @@ rte_vhost_driver_get_protocol_features(const char *path,
>>> did = vsocket->vdpa_dev_id;
>>> vdpa_dev = rte_vdpa_get_device(did);
>>> if (!vdpa_dev || !vdpa_dev->ops->get_protocol_features) {
>>> - *protocol_features = VHOST_USER_PROTOCOL_FEATURES;
>>> + *protocol_features = vsocket->protocol_features;
>>> goto unlock_exit;
>>> }
>>>
>>> @@ -744,7 +746,7 @@ rte_vhost_driver_get_protocol_features(const char *path,
>>> goto unlock_exit;
>>> }
>>>
>>> - *protocol_features = VHOST_USER_PROTOCOL_FEATURES
>>> + *protocol_features = vsocket->protocol_features
>>> & vdpa_protocol_features;
>>>
>>> unlock_exit:
>>> @@ -863,6 +865,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
>>> vsocket->use_builtin_virtio_net = true;
>>> vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES;
>>> vsocket->features = VIRTIO_NET_SUPPORTED_FEATURES;
>>> + vsocket->protocol_features = VHOST_USER_PROTOCOL_FEATURES;
>>>
>>> /* Dequeue zero copy can't assure descriptors returned in order */
>>> if (vsocket->dequeue_zero_copy) {
>>> @@ -875,6 +878,18 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
>>> vsocket->features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
>>> }
>>>
>>> + if (!(flags & RTE_VHOST_USER_POSTCOPY_SUPPORT)) {
>>> + vsocket->protocol_features &=
>>> + ~(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT);
>>> + } else {
>>> +#ifndef RTE_LIBRTE_VHOST_POSTCOPY
>>> + RTE_LOG(ERR, VHOST_CONFIG,
>>> + "Postcopy requested but not compiled\n");
>>> + ret = -1;
>>> + goto out_mutex;
>>> +#endif
>>> + }
>>> +
>>> if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
>>> vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
>>> if (vsocket->reconnect && reconn_tid == 0) {
>>>
>>
>>
@@ -106,6 +106,14 @@ The following is an overview of some key Vhost API functions:
Enabling this flag with these Qemu version results in Qemu being blocked
when multiple queue pairs are declared.
+ - ``RTE_VHOST_USER_POSTCOPY_SUPPORT``
+
+ Postcopy live-migration support will be enabled when this flag is set.
+ It is disabled by default.
+
+ Enabling this flag should only be done when the calling application does
+ not pre-fault the guest shared memory, otherwise migration would fail.
+
* ``rte_vhost_driver_set_features(path, features)``
This function sets the feature bits the vhost-user driver supports. The
@@ -28,6 +28,7 @@ extern "C" {
#define RTE_VHOST_USER_NO_RECONNECT (1ULL << 1)
#define RTE_VHOST_USER_DEQUEUE_ZERO_COPY (1ULL << 2)
#define RTE_VHOST_USER_IOMMU_SUPPORT (1ULL << 3)
+#define RTE_VHOST_USER_POSTCOPY_SUPPORT (1ULL << 4)
/** Protocol features. */
#ifndef VHOST_USER_PROTOCOL_F_MQ
@@ -51,6 +51,8 @@ struct vhost_user_socket {
uint64_t supported_features;
uint64_t features;
+ uint64_t protocol_features;
+
/*
* Device id to identify a specific backend device.
* It's set to -1 for the default software implementation.
@@ -731,7 +733,7 @@ rte_vhost_driver_get_protocol_features(const char *path,
did = vsocket->vdpa_dev_id;
vdpa_dev = rte_vdpa_get_device(did);
if (!vdpa_dev || !vdpa_dev->ops->get_protocol_features) {
- *protocol_features = VHOST_USER_PROTOCOL_FEATURES;
+ *protocol_features = vsocket->protocol_features;
goto unlock_exit;
}
@@ -744,7 +746,7 @@ rte_vhost_driver_get_protocol_features(const char *path,
goto unlock_exit;
}
- *protocol_features = VHOST_USER_PROTOCOL_FEATURES
+ *protocol_features = vsocket->protocol_features
& vdpa_protocol_features;
unlock_exit:
@@ -863,6 +865,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
vsocket->use_builtin_virtio_net = true;
vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES;
vsocket->features = VIRTIO_NET_SUPPORTED_FEATURES;
+ vsocket->protocol_features = VHOST_USER_PROTOCOL_FEATURES;
/* Dequeue zero copy can't assure descriptors returned in order */
if (vsocket->dequeue_zero_copy) {
@@ -875,6 +878,18 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
vsocket->features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
}
+ if (!(flags & RTE_VHOST_USER_POSTCOPY_SUPPORT)) {
+ vsocket->protocol_features &=
+ ~(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT);
+ } else {
+#ifndef RTE_LIBRTE_VHOST_POSTCOPY
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Postcopy requested but not compiled\n");
+ ret = -1;
+ goto out_mutex;
+#endif
+ }
+
if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
if (vsocket->reconnect && reconn_tid == 0) {