On Wed, 26 Mar 2025 at 13:32, Cédric Le Goater <clg@redhat.com> wrote:
> Gather all VFIO migration related declarations into
> "vfio-migration-internal.h" to reduce exposure of VFIO internals in
> "hw/vfio/vfio-common.h".
>
> Cc: Kirti Wankhede <kwankhede@nvidia.com>
> Cc: Avihai Horon <avihaih@nvidia.com>
> Signed-off-by: Cédric Le Goater <clg@redhat.com>
> ---
> hw/vfio/vfio-migration-internal.h | 72 +++++++++++++++++++++++++++++++
> include/hw/vfio/vfio-common.h | 52 +---------------------
> hw/vfio/common.c | 1 +
> hw/vfio/migration-multifd.c | 1 +
> hw/vfio/migration.c | 1 +
> hw/vfio/pci.c | 1 +
> 6 files changed, 77 insertions(+), 51 deletions(-)
> create mode 100644 hw/vfio/vfio-migration-internal.h
>
> diff --git a/hw/vfio/vfio-migration-internal.h b/hw/vfio/vfio-migration-internal.h
> new file mode 100644
> index 0000000000000000000000000000000000000000..ab6a1bad9b513aa61557905e72e5c6b264372276
> --- /dev/null
> +++ b/hw/vfio/vfio-migration-internal.h
> @@ -0,0 +1,72 @@
> +/*
> + * VFIO migration
> + *
> + * Copyright Red Hat, Inc. 2025
> + *
> + * SPDX-License-Identifier: GPL-2.0-or-later
> + */
> +
> +#ifndef HW_VFIO_VFIO_MIGRATION_INTERNAL_H
> +#define HW_VFIO_VFIO_MIGRATION_INTERNAL_H
> +
> +#ifdef CONFIG_LINUX
> +#include <linux/vfio.h>
> +#endif
> +
> +#include "qemu/typedefs.h"
> +#include "qemu/notify.h"
> +
> +/*
> + * Flags to be used as unique delimiters for VFIO devices in the migration
> + * stream. These flags are composed as:
> + * 0xffffffff => MSB 32-bit all 1s
> + * 0xef10 => Magic ID, represents emulated (virtual) function IO
> + * 0x0000 => 16-bits reserved for flags
> + *
> + * The beginning of state information is marked by _DEV_CONFIG_STATE,
> + * _DEV_SETUP_STATE, or _DEV_DATA_STATE, respectively. The end of a
> + * certain state information is marked by _END_OF_STATE.
> + */
> +#define VFIO_MIG_FLAG_END_OF_STATE (0xffffffffef100001ULL)
> +#define VFIO_MIG_FLAG_DEV_CONFIG_STATE (0xffffffffef100002ULL)
> +#define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL)
> +#define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL)
> +#define VFIO_MIG_FLAG_DEV_INIT_DATA_SENT (0xffffffffef100005ULL)
> +
> +typedef struct VFIODevice VFIODevice;
> +typedef struct VFIOMultifd VFIOMultifd;
> +
> +typedef struct VFIOMigration {
> + struct VFIODevice *vbasedev;
> + VMChangeStateEntry *vm_state;
> + NotifierWithReturn migration_state;
> + uint32_t device_state;
> + int data_fd;
> + void *data_buffer;
> + size_t data_buffer_size;
> + uint64_t mig_flags;
> + uint64_t precopy_init_size;
> + uint64_t precopy_dirty_size;
> + bool multifd_transfer;
> + VFIOMultifd *multifd;
> + bool initial_data_sent;
> +
> + bool event_save_iterate_started;
> + bool event_precopy_empty_hit;
> +} VFIOMigration;
> +
> +bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp);
> +void vfio_migration_exit(VFIODevice *vbasedev);
> +int vfio_save_device_config_state(QEMUFile *f, void *opaque, Error **errp);
> +int vfio_load_device_config_state(QEMUFile *f, void *opaque);
> +
> +#ifdef CONFIG_LINUX
> +int vfio_migration_set_state(VFIODevice *vbasedev,
> + enum vfio_device_mig_state new_state,
> + enum vfio_device_mig_state recover_state,
> + Error **errp);
> +#endif
> +
> +void vfio_migration_add_bytes_transferred(unsigned long val);
> +
> +#endif /* HW_VFIO_VFIO_MIGRATION_INTERNAL_H */
> diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
> index 8bb206d755adab4efee32a173c920d9aa4833685..920ad154b4f82b9c244b5b9e8da44f7583e5ead0 100644
> --- a/include/hw/vfio/vfio-common.h
> +++ b/include/hw/vfio/vfio-common.h
> @@ -23,7 +23,6 @@
>
> #include "exec/memory.h"
> #include "qemu/queue.h"
> -#include "qemu/notify.h"
> #include "ui/console.h"
> #include "hw/display/ramfb.h"
> #ifdef CONFIG_LINUX
> @@ -36,23 +35,6 @@
>
> #define VFIO_MSG_PREFIX "vfio %s: "
>
> -/*
> - * Flags to be used as unique delimiters for VFIO devices in the migration
> - * stream. These flags are composed as:
> - * 0xffffffff => MSB 32-bit all 1s
> - * 0xef10 => Magic ID, represents emulated (virtual) function IO
> - * 0x0000 => 16-bits reserved for flags
> - *
> - * The beginning of state information is marked by _DEV_CONFIG_STATE,
> - * _DEV_SETUP_STATE, or _DEV_DATA_STATE, respectively. The end of a
> - * certain state information is marked by _END_OF_STATE.
> - */
> -#define VFIO_MIG_FLAG_END_OF_STATE (0xffffffffef100001ULL)
> -#define VFIO_MIG_FLAG_DEV_CONFIG_STATE (0xffffffffef100002ULL)
> -#define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL)
> -#define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL)
> -#define VFIO_MIG_FLAG_DEV_INIT_DATA_SENT (0xffffffffef100005ULL)
> -
> enum {
> VFIO_DEVICE_TYPE_PCI = 0,
> VFIO_DEVICE_TYPE_PLATFORM = 1,
> @@ -78,27 +60,6 @@ typedef struct VFIORegion {
> uint8_t nr; /* cache the region number for debug */
> } VFIORegion;
>
> -typedef struct VFIOMultifd VFIOMultifd;
> -
> -typedef struct VFIOMigration {
> - struct VFIODevice *vbasedev;
> - VMChangeStateEntry *vm_state;
> - NotifierWithReturn migration_state;
> - uint32_t device_state;
> - int data_fd;
> - void *data_buffer;
> - size_t data_buffer_size;
> - uint64_t mig_flags;
> - uint64_t precopy_init_size;
> - uint64_t precopy_dirty_size;
> - bool multifd_transfer;
> - VFIOMultifd *multifd;
> - bool initial_data_sent;
> -
> - bool event_save_iterate_started;
> - bool event_precopy_empty_hit;
> -} VFIOMigration;
> -
> struct VFIOGroup;
>
> typedef struct VFIOContainer {
> @@ -136,6 +97,7 @@ typedef struct VFIOIOMMUFDContainer {
> OBJECT_DECLARE_SIMPLE_TYPE(VFIOIOMMUFDContainer, VFIO_IOMMU_IOMMUFD);
>
> typedef struct VFIODeviceOps VFIODeviceOps;
> +typedef struct VFIOMigration VFIOMigration;
>
> typedef struct VFIODevice {
> QLIST_ENTRY(VFIODevice) next;
> @@ -290,13 +252,9 @@ extern VFIODeviceList vfio_device_list;
> extern const MemoryListener vfio_memory_listener;
> extern int vfio_kvm_device_fd;
>
> -void vfio_migration_add_bytes_transferred(unsigned long val);
> bool vfio_device_state_is_running(VFIODevice *vbasedev);
> bool vfio_device_state_is_precopy(VFIODevice *vbasedev);
>
> -int vfio_save_device_config_state(QEMUFile *f, void *opaque, Error **errp);
> -int vfio_load_device_config_state(QEMUFile *f, void *opaque);
> -
> #ifdef CONFIG_LINUX
> int vfio_get_region_info(VFIODevice *vbasedev, int index,
> struct vfio_region_info **info);
> @@ -311,16 +269,8 @@ struct vfio_info_cap_header *
> vfio_get_device_info_cap(struct vfio_device_info *info, uint16_t id);
> struct vfio_info_cap_header *
> vfio_get_cap(void *ptr, uint32_t cap_offset, uint16_t id);
> -
> -int vfio_migration_set_state(VFIODevice *vbasedev,
> - enum vfio_device_mig_state new_state,
> - enum vfio_device_mig_state recover_state,
> - Error **errp);
> #endif
>
> -bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp);
> -void vfio_migration_exit(VFIODevice *vbasedev);
> -
> int vfio_bitmap_alloc(VFIOBitmap *vbmap, hwaddr size);
> bool vfio_devices_all_dirty_tracking_started(
> const VFIOContainerBase *bcontainer);
> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> index a4ec60e018df7dc9bbbeac27120907b0f7bf2f83..9bbd798e3e9e0b7a7620d4b1f34f1ef5e8ac42c1 100644
> --- a/hw/vfio/common.c
> +++ b/hw/vfio/common.c
> @@ -44,6 +44,7 @@
> #include "migration/qemu-file.h"
> #include "system/tcg.h"
> #include "system/tpm.h"
> +#include "vfio-migration-internal.h"
>
> VFIODeviceList vfio_device_list =
> QLIST_HEAD_INITIALIZER(vfio_device_list);
> diff --git a/hw/vfio/migration-multifd.c b/hw/vfio/migration-multifd.c
> index 09aa57f5f890f37f7e36c857fd813f55b1da2fce..1eba74beb0f207804e9c80935eacb0de19589b85 100644
> --- a/hw/vfio/migration-multifd.c
> +++ b/hw/vfio/migration-multifd.c
> @@ -22,6 +22,7 @@
> #include "io/channel-buffer.h"
> #include "migration/qemu-file.h"
> #include "migration-multifd.h"
> +#include "vfio-migration-internal.h"
> #include "trace.h"
>
> #define VFIO_DEVICE_STATE_CONFIG_STATE (1)
> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
> index 96d294794bc3056baa3b0b4e23488402db5de797..2a72a8e07542096276cc7c386359ad375e7d24c8 100644
> --- a/hw/vfio/migration.c
> +++ b/hw/vfio/migration.c
> @@ -31,6 +31,7 @@
> #include "pci.h"
> #include "trace.h"
> #include "hw/hw.h"
> +#include "vfio-migration-internal.h"
>
> /*
> * This is an arbitrary size based on migration of mlx5 devices, where typically
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 7f1532fbed9aed2eae2c98f6fd79a9056ff1e84f..158deca06cb240622a254f5059c47873e5fcc7de 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -44,6 +44,7 @@
> #include "migration/blocker.h"
> #include "migration/qemu-file.h"
> #include "system/iommufd.h"
> +#include "vfio-migration-internal.h"
>
> #define TYPE_VFIO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
>
> --
* Looks okay.
Reviewed-by: Prasad Pandit <pjp@fedoraproject.org>
Thank you.
---
- Prasad