Background snapshot allows creating a snapshot of the VM while it's
running and keeping it small by not including dirty RAM pages.

The way it works is by first stopping the VM, saving the non-iterable
devices' state and then starting the VM and saving the RAM while write
protecting it with UFFD. The resulting snapshot represents the VM state
at snapshot start.

VFIO migration is not compatible with background snapshot.
First of all, VFIO device state is not even saved in background snapshot
because only non-iterable device state is saved. But even if it was
saved, after starting the VM, a VFIO device could dirty pages without it
being detected by UFFD write protection. This would corrupt the
snapshot, as the RAM in it would not represent the RAM at snapshot
start.

To prevent this and to be explicit about supported features, block VFIO
migration with background snapshot: Fail setting background snapshot
capability if a VFIO device is present, and add a migration blocker if a
VFIO device is added when background snapshot capability is on.

Signed-off-by: Avihai Horon <avihaih@nvidia.com>
---
 include/hw/vfio/vfio-common.h |  2 ++
 migration/migration.h         |  1 +
 hw/vfio/common.c              | 42 +++++++++++++++++++++++++++++++++++
 hw/vfio/migration.c           |  6 +++++
 migration/options.c           |  7 ++++++
 migration/target.c            |  9 ++++++++
 6 files changed, 67 insertions(+)

diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index c0b58f2bb7..bb94f320f1 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -229,6 +229,8 @@ int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp);
 void vfio_unblock_multiple_devices_migration(void);
 int vfio_block_postcopy_migration(VFIODevice *vbasedev, Error **errp);
 void vfio_unblock_postcopy_migration(void);
+int vfio_block_background_snapshot(VFIODevice *vbasedev, Error **errp);
+void vfio_unblock_background_snapshot(void);
 bool vfio_viommu_preset(VFIODevice *vbasedev);
 int64_t vfio_mig_bytes_transferred(void);
 void vfio_reset_bytes_transferred(void);
diff --git a/migration/migration.h b/migration/migration.h
index 21a6423408..3077ed430b 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -516,6 +516,7 @@ void migration_populate_vfio_info(MigrationInfo *info);
 void migration_reset_vfio_bytes_transferred(void);
 bool migration_vfio_mig_active(void);
 void migration_vfio_unblock_postcopy_migration(void);
+void migration_vfio_unblock_background_snapshot(void);
 void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page);
 
 #endif
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 7461194b2b..4f6bc40cc0 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -345,6 +345,7 @@ static int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
 
 static Error *multiple_devices_migration_blocker;
 static Error *postcopy_migration_blocker;
+static Error *background_snapshot_blocker;
 
 static unsigned int vfio_migratable_devices_num(void)
 {
@@ -470,6 +471,47 @@ void vfio_unblock_postcopy_migration(void)
     postcopy_migration_blocker = NULL;
 }
 
+int vfio_block_background_snapshot(VFIODevice *vbasedev, Error **errp)
+{
+    int ret;
+
+    if (!migrate_background_snapshot()) {
+        return 0;
+    }
+
+    if (vbasedev->enable_migration == ON_OFF_AUTO_ON) {
+        error_setg(errp,
+                   "VFIO migration is not compatible with background snapshot");
+        return -EINVAL;
+    }
+
+    if (background_snapshot_blocker) {
+        return 0;
+    }
+
+    error_setg(&background_snapshot_blocker,
+               "VFIO migration is not compatible with background snapshot");
+    ret = migrate_add_blocker(background_snapshot_blocker, errp);
+    if (ret < 0) {
+        error_free(background_snapshot_blocker);
+        background_snapshot_blocker = NULL;
+    }
+
+    return ret;
+}
+
+void vfio_unblock_background_snapshot(void)
+{
+    if (!background_snapshot_blocker ||
+        (vfio_migratable_devices_num() && migrate_background_snapshot())) {
+        return;
+    }
+
+    migrate_del_blocker(background_snapshot_blocker);
+    error_free(background_snapshot_blocker);
+    background_snapshot_blocker = NULL;
+}
+
 bool vfio_mig_active(void)
 {
     return vfio_migratable_devices_num();
diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 76406e9ae9..adf98ac8e3 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -857,6 +857,7 @@ static void vfio_migration_deinit(VFIODevice *vbasedev)
     vfio_migration_free(vbasedev);
     vfio_unblock_multiple_devices_migration();
     vfio_unblock_postcopy_migration();
+    vfio_unblock_background_snapshot();
 }
 
 static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp)
@@ -945,6 +946,11 @@ bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
         goto out_deinit;
     }
 
+    ret = vfio_block_background_snapshot(vbasedev, errp);
+    if (ret) {
+        goto out_deinit;
+    }
+
     if (vfio_viommu_preset(vbasedev)) {
         error_setg(&err, "%s: Migration is currently not supported "
                    "with vIOMMU enabled", vbasedev->name);
diff --git a/migration/options.c b/migration/options.c
index e201053563..2e13363de6 100644
--- a/migration/options.c
+++ b/migration/options.c
@@ -537,6 +537,12 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp)
                 return false;
             }
         }
+
+        if (migration_vfio_mig_active()) {
+            error_setg(errp, "Background-snapshot is not compatible with VFIO "
+                             "migration");
+            return false;
+        }
     }
 
 #ifdef CONFIG_LINUX
@@ -625,6 +631,7 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp)
 static void migration_caps_remove_blockers(void)
 {
     migration_vfio_unblock_postcopy_migration();
+    migration_vfio_unblock_background_snapshot();
 }
 
 bool migrate_cap_set(int cap, bool value, Error **errp)
diff --git a/migration/target.c b/migration/target.c
index 690ecb4dd5..c2be0b39db 100644
--- a/migration/target.c
+++ b/migration/target.c
@@ -37,6 +37,11 @@ void migration_vfio_unblock_postcopy_migration(void)
 {
     vfio_unblock_postcopy_migration();
 }
+
+void migration_vfio_unblock_background_snapshot(void)
+{
+    vfio_unblock_background_snapshot();
+}
 #else
 void migration_populate_vfio_info(MigrationInfo *info)
 {
@@ -54,4 +59,8 @@ bool migration_vfio_mig_active(void)
 void migration_vfio_unblock_postcopy_migration()
 {
 }
+
+void migration_vfio_unblock_background_snapshot(void)
+{
+}
 #endif
-- 
2.26.3