On 2025/11/6 12:20, Zhenzhong Duan wrote:
> With default config, kernel VFIO IOMMU type1 driver limits dirty bitmap to
> 256MB for unmap_bitmap ioctl so the maximum guest memory region is no more
> than 8TB size for the ioctl to succeed.
>
> Be conservative here to limit total guest memory to max value supported
> by unmap_bitmap ioctl or else add a migration blocker. IOMMUFD backend
> doesn't have such limit, one can use it if there is a need to migrate such
> large VM.
>
> Suggested-by: Yi Liu <yi.l.liu@intel.com>
> Signed-off-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
> ---
Reviewed-by: Yi Liu <yi.l.liu@intel.com>
> hw/vfio/migration.c | 34 ++++++++++++++++++++++++++++++++++
> 1 file changed, 34 insertions(+)
>
> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
> index 4c06e3db93..86e5b7ab55 100644
> --- a/hw/vfio/migration.c
> +++ b/hw/vfio/migration.c
> @@ -16,6 +16,7 @@
> #include <sys/ioctl.h>
>
> #include "system/runstate.h"
> +#include "hw/boards.h"
> #include "hw/vfio/vfio-device.h"
> #include "hw/vfio/vfio-migration.h"
> #include "migration/misc.h"
> @@ -1152,6 +1153,32 @@ static bool vfio_viommu_preset(VFIODevice *vbasedev)
> return vbasedev->bcontainer->space->as != &address_space_memory;
> }
>
> +static bool vfio_dirty_tracking_exceed_limit(VFIODevice *vbasedev)
> +{
> + VFIOContainer *bcontainer = vbasedev->bcontainer;
> + uint64_t max_size, page_size;
> +
> + if (!bcontainer->dirty_pages_supported) {
> + return false;
> + }
> +
> + /*
> + * VFIO IOMMU type1 driver has limitation of bitmap size on unmap_bitmap
> + * ioctl(), calculate the limit and compare with guest memory size to
> + * catch dirty tracking failure early.
> + *
> + * This limit is 8TB with default kernel and QEMU config, we are a bit
> + * conservative here as VM memory layout may be nonconsecutive or VM
> + * can run with vIOMMU enabled so the limitation could be relaxed. One
> + * can also switch to use IOMMUFD backend if there is a need to migrate
> + * large VM.
> + */
> + page_size = 1 << ctz64(bcontainer->dirty_pgsizes);
> + max_size = bcontainer->max_dirty_bitmap_size * BITS_PER_BYTE * page_size;
> +
> + return current_machine->ram_size > max_size;
> +}
> +
> /*
> * Return true when either migration initialized or blocker registered.
> * Currently only return false when adding blocker fails which will
> @@ -1193,6 +1220,13 @@ bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
> goto add_blocker;
> }
>
> + if (vfio_dirty_tracking_exceed_limit(vbasedev)) {
> + error_setg(&err, "%s: Migration is currently not supported with "
> + "large memory VM due to dirty tracking limitation in "
> + "backend", vbasedev->name);
> + goto add_blocker;
> + }
> +
> warn_report("%s: VFIO device doesn't support device and "
> "IOMMU dirty tracking", vbasedev->name);
> }