1. Patchew
  2. QEMU
  3. [PATCH v7 0/5] support inflight migration
  4. View patch

[PATCH v7 5/5] vhost-user-blk: support inter-host inflight migration

Alexandr Moshkov posted 5 patches 3 weeks, 5 days ago
[PATCH v7 5/5] vhost-user-blk: support inter-host inflight migration
Posted by Alexandr Moshkov 3 weeks, 5 days ago 
During inter-host migration, waiting for disk requests to be drained
in the vhost-user backend can incur significant downtime.

This can be avoided if QEMU migrates the inflight region in
vhost-user-blk.
Thus, during the qemu migration, with feature flag the vhost-user
back-end can immediately stop vrings, so all in-flight requests will be
migrated to another host.

Signed-off-by: Alexandr Moshkov <dtalexundeer@yandex-team.ru>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/block/vhost-user-blk.c          | 27 +++++++++++++++++++++++++++
 include/hw/virtio/vhost-user-blk.h |  1 +
 include/hw/virtio/vhost.h          |  6 ++++++
 3 files changed, 34 insertions(+)

diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index 4d81d2dc34..c151e83677 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -353,6 +353,7 @@ static int vhost_user_blk_connect(DeviceState *dev, Error **errp)
     vhost_dev_set_config_notifier(&s->dev, &blk_ops);
 
     s->vhost_user.supports_config = true;
+    s->vhost_user.supports_inflight_migration = s->inflight_migration;
     ret = vhost_dev_init(&s->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
                          errp);
     if (ret < 0) {
@@ -568,6 +569,26 @@ static struct vhost_dev *vhost_user_blk_get_vhost(VirtIODevice *vdev)
     return &s->dev;
 }
 
+static bool vhost_user_blk_inflight_needed(void *opaque)
+{
+    struct VHostUserBlk *s = opaque;
+
+    bool inflight_migration = virtio_has_feature(s->dev.protocol_features,
+                               VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT);
+
+    return inflight_migration;
+}
+
+static const VMStateDescription vmstate_vhost_user_blk_inflight = {
+    .name = "vhost-user-blk/inflight",
+    .version_id = 1,
+    .needed = vhost_user_blk_inflight_needed,
+    .fields = (const VMStateField[]) {
+        VMSTATE_VHOST_INFLIGHT_REGION(inflight, VHostUserBlk),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static const VMStateDescription vmstate_vhost_user_blk = {
     .name = "vhost-user-blk",
     .minimum_version_id = 1,
@@ -576,6 +597,10 @@ static const VMStateDescription vmstate_vhost_user_blk = {
         VMSTATE_VIRTIO_DEVICE,
         VMSTATE_END_OF_LIST()
     },
+    .subsections = (const VMStateDescription * const []) {
+        &vmstate_vhost_user_blk_inflight,
+        NULL
+    }
 };
 
 static const Property vhost_user_blk_properties[] = {
@@ -591,6 +616,8 @@ static const Property vhost_user_blk_properties[] = {
                       VIRTIO_BLK_F_WRITE_ZEROES, true),
     DEFINE_PROP_BOOL("skip-get-vring-base-on-force-shutdown", VHostUserBlk,
                      skip_get_vring_base_on_force_shutdown, false),
+    DEFINE_PROP_BOOL("inflight-migration", VHostUserBlk,
+                     inflight_migration, false),
 };
 
 static void vhost_user_blk_class_init(ObjectClass *klass, const void *data)
diff --git a/include/hw/virtio/vhost-user-blk.h b/include/hw/virtio/vhost-user-blk.h
index 8158d4673d..1e41a2bcdf 100644
--- a/include/hw/virtio/vhost-user-blk.h
+++ b/include/hw/virtio/vhost-user-blk.h
@@ -52,6 +52,7 @@ struct VHostUserBlk {
     bool started_vu;
 
     bool skip_get_vring_base_on_force_shutdown;
+    bool inflight_migration;
 };
 
 #endif
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index 08bbb4dfe9..89817bd848 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -554,4 +554,10 @@ static inline int vhost_load_backend_state(struct vhost_dev *dev, QEMUFile *f,
 }
 #endif
 
+extern const VMStateDescription vmstate_vhost_inflight_region;
+#define VMSTATE_VHOST_INFLIGHT_REGION(_field, _state) \
+    VMSTATE_STRUCT_POINTER(_field, _state, \
+                           vmstate_vhost_inflight_region, \
+                           struct vhost_inflight)
+
 #endif
-- 
2.34.1
Re: [PATCH v7 5/5] vhost-user-blk: support inter-host inflight migration
Posted by Raphael Norwitz 3 weeks, 3 days ago 
Apologies if I've missed something obvious here - what actually skips
the GET_VRING_BASE operation here? I was expecting something like:

--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -221,8 +221,9 @@ static int vhost_user_blk_stop(VirtIODevice *vdev)
         return 0;
     }

-    force_stop = s->skip_get_vring_base_on_force_shutdown &&
-                 qemu_force_shutdown_requested();
+    force_stop = (s->skip_get_vring_base_on_force_shutdown &&
+                 qemu_force_shutdown_requested()) ||
+                 s->inflight_migration; // FIXME: runstate check?

Without setting force_stop in vhost_user_blk_stop() I don't see how
the GET_VRING_BASE operation would be skipped.

On Thu, Jan 15, 2026 at 3:13 AM Alexandr Moshkov
<dtalexundeer@yandex-team.ru> wrote:
>
> During inter-host migration, waiting for disk requests to be drained
> in the vhost-user backend can incur significant downtime.
>
> This can be avoided if QEMU migrates the inflight region in
> vhost-user-blk.
> Thus, during the qemu migration, with feature flag the vhost-user
> back-end can immediately stop vrings, so all in-flight requests will be
> migrated to another host.
>
> Signed-off-by: Alexandr Moshkov <dtalexundeer@yandex-team.ru>
> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
> ---
>  hw/block/vhost-user-blk.c          | 27 +++++++++++++++++++++++++++
>  include/hw/virtio/vhost-user-blk.h |  1 +
>  include/hw/virtio/vhost.h          |  6 ++++++
>  3 files changed, 34 insertions(+)
>
> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
> index 4d81d2dc34..c151e83677 100644
> --- a/hw/block/vhost-user-blk.c
> +++ b/hw/block/vhost-user-blk.c
> @@ -353,6 +353,7 @@ static int vhost_user_blk_connect(DeviceState *dev, Error **errp)
>      vhost_dev_set_config_notifier(&s->dev, &blk_ops);
>
>      s->vhost_user.supports_config = true;
> +    s->vhost_user.supports_inflight_migration = s->inflight_migration;
>      ret = vhost_dev_init(&s->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>                           errp);
>      if (ret < 0) {
> @@ -568,6 +569,26 @@ static struct vhost_dev *vhost_user_blk_get_vhost(VirtIODevice *vdev)
>      return &s->dev;
>  }
>
> +static bool vhost_user_blk_inflight_needed(void *opaque)
> +{
> +    struct VHostUserBlk *s = opaque;
> +
> +    bool inflight_migration = virtio_has_feature(s->dev.protocol_features,
> +                               VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT);
> +
> +    return inflight_migration;
> +}
> +
> +static const VMStateDescription vmstate_vhost_user_blk_inflight = {
> +    .name = "vhost-user-blk/inflight",
> +    .version_id = 1,
> +    .needed = vhost_user_blk_inflight_needed,
> +    .fields = (const VMStateField[]) {
> +        VMSTATE_VHOST_INFLIGHT_REGION(inflight, VHostUserBlk),
> +        VMSTATE_END_OF_LIST()
> +    },
> +};
> +
>  static const VMStateDescription vmstate_vhost_user_blk = {
>      .name = "vhost-user-blk",
>      .minimum_version_id = 1,
> @@ -576,6 +597,10 @@ static const VMStateDescription vmstate_vhost_user_blk = {
>          VMSTATE_VIRTIO_DEVICE,
>          VMSTATE_END_OF_LIST()
>      },
> +    .subsections = (const VMStateDescription * const []) {
> +        &vmstate_vhost_user_blk_inflight,
> +        NULL
> +    }
>  };
>
>  static const Property vhost_user_blk_properties[] = {
> @@ -591,6 +616,8 @@ static const Property vhost_user_blk_properties[] = {
>                        VIRTIO_BLK_F_WRITE_ZEROES, true),
>      DEFINE_PROP_BOOL("skip-get-vring-base-on-force-shutdown", VHostUserBlk,
>                       skip_get_vring_base_on_force_shutdown, false),
> +    DEFINE_PROP_BOOL("inflight-migration", VHostUserBlk,
> +                     inflight_migration, false),
>  };
>
>  static void vhost_user_blk_class_init(ObjectClass *klass, const void *data)
> diff --git a/include/hw/virtio/vhost-user-blk.h b/include/hw/virtio/vhost-user-blk.h
> index 8158d4673d..1e41a2bcdf 100644
> --- a/include/hw/virtio/vhost-user-blk.h
> +++ b/include/hw/virtio/vhost-user-blk.h
> @@ -52,6 +52,7 @@ struct VHostUserBlk {
>      bool started_vu;
>
>      bool skip_get_vring_base_on_force_shutdown;
> +    bool inflight_migration;
>  };
>
>  #endif
> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
> index 08bbb4dfe9..89817bd848 100644
> --- a/include/hw/virtio/vhost.h
> +++ b/include/hw/virtio/vhost.h
> @@ -554,4 +554,10 @@ static inline int vhost_load_backend_state(struct vhost_dev *dev, QEMUFile *f,
>  }
>  #endif
>
> +extern const VMStateDescription vmstate_vhost_inflight_region;
> +#define VMSTATE_VHOST_INFLIGHT_REGION(_field, _state) \
> +    VMSTATE_STRUCT_POINTER(_field, _state, \
> +                           vmstate_vhost_inflight_region, \
> +                           struct vhost_inflight)
> +
>  #endif
> --
> 2.34.1
>
>
Re: [PATCH v7 5/5] vhost-user-blk: support inter-host inflight migration
Posted by Alexandr Moshkov 3 weeks, 1 day ago 
On 1/17/26 00:50, Raphael Norwitz wrote:
> Apologies if I've missed something obvious here - what actually skips
> the GET_VRING_BASE operation here? I was expecting something like:
>
> --- a/hw/block/vhost-user-blk.c
> +++ b/hw/block/vhost-user-blk.c
> @@ -221,8 +221,9 @@ static int vhost_user_blk_stop(VirtIODevice *vdev)
>           return 0;
>       }
>
> -    force_stop = s->skip_get_vring_base_on_force_shutdown &&
> -                 qemu_force_shutdown_requested();
> +    force_stop = (s->skip_get_vring_base_on_force_shutdown &&
> +                 qemu_force_shutdown_requested()) ||
> +                 s->inflight_migration; // FIXME: runstate check?
>
> Without setting force_stop in vhost_user_blk_stop() I don't see how
> the GET_VRING_BASE operation would be skipped.

Nothing skip GET_VRING_BASE) In the first versions of the patch, this 
was done, but then I realized that this was the wrong way, because the 
last_avail_idx and avail_idx counters may be incorrect after migration.

Therefore, it is now implemented that on GET_VRING_BASE the backend 
needs to either wait for all in-flight requests or not (depending on 
protocol_feature).

> On Thu, Jan 15, 2026 at 3:13 AM Alexandr Moshkov
> <dtalexundeer@yandex-team.ru> wrote:
>> During inter-host migration, waiting for disk requests to be drained
>> in the vhost-user backend can incur significant downtime.
>>
>> This can be avoided if QEMU migrates the inflight region in
>> vhost-user-blk.
>> Thus, during the qemu migration, with feature flag the vhost-user
>> back-end can immediately stop vrings, so all in-flight requests will be
>> migrated to another host.
>>
>> Signed-off-by: Alexandr Moshkov<dtalexundeer@yandex-team.ru>
>> Reviewed-by: Stefan Hajnoczi<stefanha@redhat.com>
>> ---
>>   hw/block/vhost-user-blk.c          | 27 +++++++++++++++++++++++++++
>>   include/hw/virtio/vhost-user-blk.h |  1 +
>>   include/hw/virtio/vhost.h          |  6 ++++++
>>   3 files changed, 34 insertions(+)
>>
>> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
>> index 4d81d2dc34..c151e83677 100644
>> --- a/hw/block/vhost-user-blk.c
>> +++ b/hw/block/vhost-user-blk.c
>> @@ -353,6 +353,7 @@ static int vhost_user_blk_connect(DeviceState *dev, Error **errp)
>>       vhost_dev_set_config_notifier(&s->dev, &blk_ops);
>>
>>       s->vhost_user.supports_config = true;
>> +    s->vhost_user.supports_inflight_migration = s->inflight_migration;
>>       ret = vhost_dev_init(&s->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>>                            errp);
>>       if (ret < 0) {
>> @@ -568,6 +569,26 @@ static struct vhost_dev *vhost_user_blk_get_vhost(VirtIODevice *vdev)
>>       return &s->dev;
>>   }
>>
>> +static bool vhost_user_blk_inflight_needed(void *opaque)
>> +{
>> +    struct VHostUserBlk *s = opaque;
>> +
>> +    bool inflight_migration = virtio_has_feature(s->dev.protocol_features,
>> +                               VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT);
>> +
>> +    return inflight_migration;
>> +}
>> +
>> +static const VMStateDescription vmstate_vhost_user_blk_inflight = {
>> +    .name = "vhost-user-blk/inflight",
>> +    .version_id = 1,
>> +    .needed = vhost_user_blk_inflight_needed,
>> +    .fields = (const VMStateField[]) {
>> +        VMSTATE_VHOST_INFLIGHT_REGION(inflight, VHostUserBlk),
>> +        VMSTATE_END_OF_LIST()
>> +    },
>> +};
>> +
>>   static const VMStateDescription vmstate_vhost_user_blk = {
>>       .name = "vhost-user-blk",
>>       .minimum_version_id = 1,
>> @@ -576,6 +597,10 @@ static const VMStateDescription vmstate_vhost_user_blk = {
>>           VMSTATE_VIRTIO_DEVICE,
>>           VMSTATE_END_OF_LIST()
>>       },
>> +    .subsections = (const VMStateDescription * const []) {
>> +        &vmstate_vhost_user_blk_inflight,
>> +        NULL
>> +    }
>>   };
>>
>>   static const Property vhost_user_blk_properties[] = {
>> @@ -591,6 +616,8 @@ static const Property vhost_user_blk_properties[] = {
>>                         VIRTIO_BLK_F_WRITE_ZEROES, true),
>>       DEFINE_PROP_BOOL("skip-get-vring-base-on-force-shutdown", VHostUserBlk,
>>                        skip_get_vring_base_on_force_shutdown, false),
>> +    DEFINE_PROP_BOOL("inflight-migration", VHostUserBlk,
>> +                     inflight_migration, false),
>>   };
>>
>>   static void vhost_user_blk_class_init(ObjectClass *klass, const void *data)
>> diff --git a/include/hw/virtio/vhost-user-blk.h b/include/hw/virtio/vhost-user-blk.h
>> index 8158d4673d..1e41a2bcdf 100644
>> --- a/include/hw/virtio/vhost-user-blk.h
>> +++ b/include/hw/virtio/vhost-user-blk.h
>> @@ -52,6 +52,7 @@ struct VHostUserBlk {
>>       bool started_vu;
>>
>>       bool skip_get_vring_base_on_force_shutdown;
>> +    bool inflight_migration;
>>   };
>>
>>   #endif
>> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
>> index 08bbb4dfe9..89817bd848 100644
>> --- a/include/hw/virtio/vhost.h
>> +++ b/include/hw/virtio/vhost.h
>> @@ -554,4 +554,10 @@ static inline int vhost_load_backend_state(struct vhost_dev *dev, QEMUFile *f,
>>   }
>>   #endif
>>
>> +extern const VMStateDescription vmstate_vhost_inflight_region;
>> +#define VMSTATE_VHOST_INFLIGHT_REGION(_field, _state) \
>> +    VMSTATE_STRUCT_POINTER(_field, _state, \
>> +                           vmstate_vhost_inflight_region, \
>> +                           struct vhost_inflight)
>> +
>>   #endif
>> --
>> 2.34.1
>>
>>
Re: [PATCH v7 5/5] vhost-user-blk: support inter-host inflight migration
Posted by Raphael Norwitz 3 weeks ago 
ACK on your explanation. Then for the series:

Reviewed-by: Raphael Norwitz <raphael.s.norwitz@gmail.com>

On Mon, Jan 19, 2026 at 1:44 AM Alexandr Moshkov
<dtalexundeer@yandex-team.ru> wrote:
>
>
> On 1/17/26 00:50, Raphael Norwitz wrote:
>
> Apologies if I've missed something obvious here - what actually skips
> the GET_VRING_BASE operation here? I was expecting something like:
>
> --- a/hw/block/vhost-user-blk.c
> +++ b/hw/block/vhost-user-blk.c
> @@ -221,8 +221,9 @@ static int vhost_user_blk_stop(VirtIODevice *vdev)
>          return 0;
>      }
>
> -    force_stop = s->skip_get_vring_base_on_force_shutdown &&
> -                 qemu_force_shutdown_requested();
> +    force_stop = (s->skip_get_vring_base_on_force_shutdown &&
> +                 qemu_force_shutdown_requested()) ||
> +                 s->inflight_migration; // FIXME: runstate check?
>
> Without setting force_stop in vhost_user_blk_stop() I don't see how
> the GET_VRING_BASE operation would be skipped.
>
> Nothing skip GET_VRING_BASE) In the first versions of the patch, this was done, but then I realized that this was the wrong way, because the last_avail_idx and avail_idx counters may be incorrect after migration.
>
> Therefore, it is now implemented that on GET_VRING_BASE the backend needs to either wait for all in-flight requests or not (depending on protocol_feature).
>
> On Thu, Jan 15, 2026 at 3:13 AM Alexandr Moshkov
> <dtalexundeer@yandex-team.ru> wrote:
>
> During inter-host migration, waiting for disk requests to be drained
> in the vhost-user backend can incur significant downtime.
>
> This can be avoided if QEMU migrates the inflight region in
> vhost-user-blk.
> Thus, during the qemu migration, with feature flag the vhost-user
> back-end can immediately stop vrings, so all in-flight requests will be
> migrated to another host.
>
> Signed-off-by: Alexandr Moshkov <dtalexundeer@yandex-team.ru>
> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
> ---
>  hw/block/vhost-user-blk.c          | 27 +++++++++++++++++++++++++++
>  include/hw/virtio/vhost-user-blk.h |  1 +
>  include/hw/virtio/vhost.h          |  6 ++++++
>  3 files changed, 34 insertions(+)
>
> diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
> index 4d81d2dc34..c151e83677 100644
> --- a/hw/block/vhost-user-blk.c
> +++ b/hw/block/vhost-user-blk.c
> @@ -353,6 +353,7 @@ static int vhost_user_blk_connect(DeviceState *dev, Error **errp)
>      vhost_dev_set_config_notifier(&s->dev, &blk_ops);
>
>      s->vhost_user.supports_config = true;
> +    s->vhost_user.supports_inflight_migration = s->inflight_migration;
>      ret = vhost_dev_init(&s->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
>                           errp);
>      if (ret < 0) {
> @@ -568,6 +569,26 @@ static struct vhost_dev *vhost_user_blk_get_vhost(VirtIODevice *vdev)
>      return &s->dev;
>  }
>
> +static bool vhost_user_blk_inflight_needed(void *opaque)
> +{
> +    struct VHostUserBlk *s = opaque;
> +
> +    bool inflight_migration = virtio_has_feature(s->dev.protocol_features,
> +                               VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT);
> +
> +    return inflight_migration;
> +}
> +
> +static const VMStateDescription vmstate_vhost_user_blk_inflight = {
> +    .name = "vhost-user-blk/inflight",
> +    .version_id = 1,
> +    .needed = vhost_user_blk_inflight_needed,
> +    .fields = (const VMStateField[]) {
> +        VMSTATE_VHOST_INFLIGHT_REGION(inflight, VHostUserBlk),
> +        VMSTATE_END_OF_LIST()
> +    },
> +};
> +
>  static const VMStateDescription vmstate_vhost_user_blk = {
>      .name = "vhost-user-blk",
>      .minimum_version_id = 1,
> @@ -576,6 +597,10 @@ static const VMStateDescription vmstate_vhost_user_blk = {
>          VMSTATE_VIRTIO_DEVICE,
>          VMSTATE_END_OF_LIST()
>      },
> +    .subsections = (const VMStateDescription * const []) {
> +        &vmstate_vhost_user_blk_inflight,
> +        NULL
> +    }
>  };
>
>  static const Property vhost_user_blk_properties[] = {
> @@ -591,6 +616,8 @@ static const Property vhost_user_blk_properties[] = {
>                        VIRTIO_BLK_F_WRITE_ZEROES, true),
>      DEFINE_PROP_BOOL("skip-get-vring-base-on-force-shutdown", VHostUserBlk,
>                       skip_get_vring_base_on_force_shutdown, false),
> +    DEFINE_PROP_BOOL("inflight-migration", VHostUserBlk,
> +                     inflight_migration, false),
>  };
>
>  static void vhost_user_blk_class_init(ObjectClass *klass, const void *data)
> diff --git a/include/hw/virtio/vhost-user-blk.h b/include/hw/virtio/vhost-user-blk.h
> index 8158d4673d..1e41a2bcdf 100644
> --- a/include/hw/virtio/vhost-user-blk.h
> +++ b/include/hw/virtio/vhost-user-blk.h
> @@ -52,6 +52,7 @@ struct VHostUserBlk {
>      bool started_vu;
>
>      bool skip_get_vring_base_on_force_shutdown;
> +    bool inflight_migration;
>  };
>
>  #endif
> diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
> index 08bbb4dfe9..89817bd848 100644
> --- a/include/hw/virtio/vhost.h
> +++ b/include/hw/virtio/vhost.h
> @@ -554,4 +554,10 @@ static inline int vhost_load_backend_state(struct vhost_dev *dev, QEMUFile *f,
>  }
>  #endif
>
> +extern const VMStateDescription vmstate_vhost_inflight_region;
> +#define VMSTATE_VHOST_INFLIGHT_REGION(_field, _state) \
> +    VMSTATE_STRUCT_POINTER(_field, _state, \
> +                           vmstate_vhost_inflight_region, \
> +                           struct vhost_inflight)
> +
>  #endif
> --
> 2.34.1
>
>

Patchew 2.1-devel-b67e4c2

© 2016 - 2026 Red Hat, Inc.