Currently, VFIO bytes_transferred is not reset properly:
1. bytes_transferred is not reset after a VM snapshot (so a migration
following a snapshot will report incorrect value).
2. bytes_transferred is a single counter for all VFIO devices, however
upon migration failure it is reset multiple times, by each VFIO
device.
Fix it by introducing a new function vfio_reset_bytes_transferred() and
calling it during migration and snapshot start.
Remove existing bytes_transferred reset in VFIO migration state
notifier, which is not needed anymore.
Fixes: 3710586caa5d ("qapi: Add VFIO devices migration stats in Migration stats")
Signed-off-by: Avihai Horon <avihaih@nvidia.com>
---
include/hw/vfio/vfio-common.h | 1 +
migration/migration.h | 1 +
hw/vfio/migration.c | 6 +++++-
migration/migration.c | 1 +
migration/savevm.c | 1 +
migration/target.c | 17 +++++++++++++++--
6 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 3dc5f2104c..b4c28f318f 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -228,6 +228,7 @@ int vfio_block_multiple_devices_migration(Error **errp);
void vfio_unblock_multiple_devices_migration(void);
int vfio_block_giommu_migration(Error **errp);
int64_t vfio_mig_bytes_transferred(void);
+void vfio_reset_bytes_transferred(void);
#ifdef CONFIG_LINUX
int vfio_get_region_info(VFIODevice *vbasedev, int index,
diff --git a/migration/migration.h b/migration/migration.h
index c859a0d35e..a80b22b703 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -514,6 +514,7 @@ bool migration_rate_limit(void);
void migration_cancel(const Error *error);
void populate_vfio_info(MigrationInfo *info);
+void reset_vfio_bytes_transferred(void);
void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page);
#endif
diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index acbf0bb7ab..7cf143926c 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -697,7 +697,6 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data)
case MIGRATION_STATUS_CANCELLING:
case MIGRATION_STATUS_CANCELLED:
case MIGRATION_STATUS_FAILED:
- bytes_transferred = 0;
/*
* If setting the device in RUNNING state fails, the device should
* be reset. To do so, use ERROR state as a recover state.
@@ -818,6 +817,11 @@ int64_t vfio_mig_bytes_transferred(void)
return bytes_transferred;
}
+void vfio_reset_bytes_transferred(void)
+{
+ bytes_transferred = 0;
+}
+
int vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
{
int ret = -ENOTSUP;
diff --git a/migration/migration.c b/migration/migration.c
index 7653787f74..096e8191d1 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1628,6 +1628,7 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
*/
memset(&mig_stats, 0, sizeof(mig_stats));
memset(&compression_counters, 0, sizeof(compression_counters));
+ reset_vfio_bytes_transferred();
return true;
}
diff --git a/migration/savevm.c b/migration/savevm.c
index cdf4793924..95c2abf47c 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -1622,6 +1622,7 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp)
migrate_init(ms);
memset(&mig_stats, 0, sizeof(mig_stats));
memset(&compression_counters, 0, sizeof(compression_counters));
+ reset_vfio_bytes_transferred();
ms->to_dst_file = f;
qemu_mutex_unlock_iothread();
diff --git a/migration/target.c b/migration/target.c
index 00ca007f97..f39c9a8d88 100644
--- a/migration/target.c
+++ b/migration/target.c
@@ -14,12 +14,25 @@
#include "hw/vfio/vfio-common.h"
#endif
+#ifdef CONFIG_VFIO
void populate_vfio_info(MigrationInfo *info)
{
-#ifdef CONFIG_VFIO
if (vfio_mig_active()) {
info->vfio = g_malloc0(sizeof(*info->vfio));
info->vfio->transferred = vfio_mig_bytes_transferred();
}
-#endif
}
+
+void reset_vfio_bytes_transferred(void)
+{
+ vfio_reset_bytes_transferred();
+}
+#else
+void populate_vfio_info(MigrationInfo *info)
+{
+}
+
+void reset_vfio_bytes_transferred(void)
+{
+}
+#endif
--
2.26.3On 6/28/23 09:31, Avihai Horon wrote:
> Currently, VFIO bytes_transferred is not reset properly:
> 1. bytes_transferred is not reset after a VM snapshot (so a migration
> following a snapshot will report incorrect value).
> 2. bytes_transferred is a single counter for all VFIO devices, however
> upon migration failure it is reset multiple times, by each VFIO
> device.
>
> Fix it by introducing a new function vfio_reset_bytes_transferred() and
> calling it during migration and snapshot start.
>
> Remove existing bytes_transferred reset in VFIO migration state
> notifier, which is not needed anymore.
>
> Fixes: 3710586caa5d ("qapi: Add VFIO devices migration stats in Migration stats")
> Signed-off-by: Avihai Horon <avihaih@nvidia.com>
Reviewed-by: Cédric Le Goater <clg@redhat.com>
Thanks,
C.
> ---
> include/hw/vfio/vfio-common.h | 1 +
> migration/migration.h | 1 +
> hw/vfio/migration.c | 6 +++++-
> migration/migration.c | 1 +
> migration/savevm.c | 1 +
> migration/target.c | 17 +++++++++++++++--
> 6 files changed, 24 insertions(+), 3 deletions(-)
>
> diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
> index 3dc5f2104c..b4c28f318f 100644
> --- a/include/hw/vfio/vfio-common.h
> +++ b/include/hw/vfio/vfio-common.h
> @@ -228,6 +228,7 @@ int vfio_block_multiple_devices_migration(Error **errp);
> void vfio_unblock_multiple_devices_migration(void);
> int vfio_block_giommu_migration(Error **errp);
> int64_t vfio_mig_bytes_transferred(void);
> +void vfio_reset_bytes_transferred(void);
>
> #ifdef CONFIG_LINUX
> int vfio_get_region_info(VFIODevice *vbasedev, int index,
> diff --git a/migration/migration.h b/migration/migration.h
> index c859a0d35e..a80b22b703 100644
> --- a/migration/migration.h
> +++ b/migration/migration.h
> @@ -514,6 +514,7 @@ bool migration_rate_limit(void);
> void migration_cancel(const Error *error);
>
> void populate_vfio_info(MigrationInfo *info);
> +void reset_vfio_bytes_transferred(void);
> void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page);
>
> #endif
> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
> index acbf0bb7ab..7cf143926c 100644
> --- a/hw/vfio/migration.c
> +++ b/hw/vfio/migration.c
> @@ -697,7 +697,6 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data)
> case MIGRATION_STATUS_CANCELLING:
> case MIGRATION_STATUS_CANCELLED:
> case MIGRATION_STATUS_FAILED:
> - bytes_transferred = 0;
> /*
> * If setting the device in RUNNING state fails, the device should
> * be reset. To do so, use ERROR state as a recover state.
> @@ -818,6 +817,11 @@ int64_t vfio_mig_bytes_transferred(void)
> return bytes_transferred;
> }
>
> +void vfio_reset_bytes_transferred(void)
> +{
> + bytes_transferred = 0;
> +}
> +
> int vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
> {
> int ret = -ENOTSUP;
> diff --git a/migration/migration.c b/migration/migration.c
> index 7653787f74..096e8191d1 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -1628,6 +1628,7 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
> */
> memset(&mig_stats, 0, sizeof(mig_stats));
> memset(&compression_counters, 0, sizeof(compression_counters));
> + reset_vfio_bytes_transferred();
>
> return true;
> }
> diff --git a/migration/savevm.c b/migration/savevm.c
> index cdf4793924..95c2abf47c 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -1622,6 +1622,7 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp)
> migrate_init(ms);
> memset(&mig_stats, 0, sizeof(mig_stats));
> memset(&compression_counters, 0, sizeof(compression_counters));
> + reset_vfio_bytes_transferred();
> ms->to_dst_file = f;
>
> qemu_mutex_unlock_iothread();
> diff --git a/migration/target.c b/migration/target.c
> index 00ca007f97..f39c9a8d88 100644
> --- a/migration/target.c
> +++ b/migration/target.c
> @@ -14,12 +14,25 @@
> #include "hw/vfio/vfio-common.h"
> #endif
>
> +#ifdef CONFIG_VFIO
> void populate_vfio_info(MigrationInfo *info)
> {
> -#ifdef CONFIG_VFIO
> if (vfio_mig_active()) {
> info->vfio = g_malloc0(sizeof(*info->vfio));
> info->vfio->transferred = vfio_mig_bytes_transferred();
> }
> -#endif
> }
> +
> +void reset_vfio_bytes_transferred(void)
> +{
> + vfio_reset_bytes_transferred();
> +}
> +#else
> +void populate_vfio_info(MigrationInfo *info)
> +{
> +}
> +
> +void reset_vfio_bytes_transferred(void)
> +{
> +}
> +#endif
© 2016 - 2026 Red Hat, Inc.