From nobody Fri May 17 00:57:11 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1520229458615672.5097192305323; Sun, 4 Mar 2018 21:57:38 -0800 (PST) Received: from localhost ([::1]:47275 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj7o-0005u6-4C for importer@patchew.org; Mon, 05 Mar 2018 00:57:28 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:39325) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj63-0004zx-7S for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:40 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1esj60-0008FJ-FT for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:39 -0500 Received: from mga06.intel.com ([134.134.136.31]:54640) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1esj60-0008E6-6c for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:36 -0500 Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 04 Mar 2018 21:55:30 -0800 Received: from yzhang13-kvm.sh.intel.com ([10.239.36.9]) by fmsmga005.fm.intel.com with ESMTP; 04 Mar 2018 21:55:29 -0800 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,426,1515484800"; d="scan'208";a="208826738" From: Yulei Zhang To: qemu-devel@nongnu.org Date: Mon, 5 Mar 2018 14:00:50 +0800 Message-Id: <1520229653-10658-2-git-send-email-yulei.zhang@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> References: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 134.134.136.31 Subject: [Qemu-devel] [PATCH V3 1/4] vfio: introduce a new VFIO subregion for mdev device migration support X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Yulei Zhang , kevin.tian@intel.com, alex.williamson@redhat.com, kwankhede@nvidia.com, zhenyuw@linux.intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" New VFIO sub region VFIO_REGION_SUBTYPE_DEVICE_STATE is added to fetch and restore the status of mdev device vGPU during the live migration. Signed-off-by: Yulei Zhang --- hw/vfio/pci.c | 14 +++++++++++++- hw/vfio/pci.h | 1 + linux-headers/linux/vfio.h | 9 ++++++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 31e1edf..2fe20e4 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -37,6 +37,7 @@ =20 static void vfio_disable_interrupts(VFIOPCIDevice *vdev); static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled); +static VMStateDescription vfio_pci_vmstate; =20 /* * Disabling BAR mmaping can be slow, but toggling it around INTx can @@ -2813,6 +2814,17 @@ static void vfio_realize(PCIDevice *pdev, Error **er= rp) vfio_vga_quirk_setup(vdev); } =20 + struct vfio_region_info *device_state; + /* device state region setup */ + if (!vfio_get_dev_region_info(&vdev->vbasedev, + VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, + VFIO_REGION_SUBTYPE_DEVICE_STATE, &device_state)) { + memcpy(&vdev->device_state, device_state, + sizeof(struct vfio_region_info)); + g_free(device_state); + vfio_pci_vmstate.unmigratable =3D 0; + } + for (i =3D 0; i < PCI_ROM_SLOT; i++) { vfio_bar_quirk_setup(vdev, i); } @@ -2994,7 +3006,7 @@ static Property vfio_pci_dev_properties[] =3D { DEFINE_PROP_END_OF_LIST(), }; =20 -static const VMStateDescription vfio_pci_vmstate =3D { +static VMStateDescription vfio_pci_vmstate =3D { .name =3D "vfio-pci", .unmigratable =3D 1, }; diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h index a8366bb..6a1d26e 100644 --- a/hw/vfio/pci.h +++ b/hw/vfio/pci.h @@ -116,6 +116,7 @@ typedef struct VFIOPCIDevice { VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */ VFIOVGA *vga; /* 0xa0000, 0x3b0, 0x3c0 */ void *igd_opregion; + struct vfio_region_info device_state; PCIHostDeviceAddress host; EventNotifier err_notifier; EventNotifier req_notifier; diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h index 4e7ab4c..c3b8e4a 100644 --- a/linux-headers/linux/vfio.h +++ b/linux-headers/linux/vfio.h @@ -296,9 +296,12 @@ struct vfio_region_info_cap_type { #define VFIO_REGION_TYPE_PCI_VENDOR_MASK (0xffff) =20 /* 8086 Vendor sub-types */ -#define VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION (1) -#define VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG (2) -#define VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG (3) +#define VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION (1) +#define VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG (2) +#define VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG (3) + +/* Mdev sub-type for device state save and restore */ +#define VFIO_REGION_SUBTYPE_DEVICE_STATE (4) =20 /** * VFIO_DEVICE_GET_IRQ_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 9, --=20 2.7.4 From nobody Fri May 17 00:57:11 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1520229456808847.253110536699; Sun, 4 Mar 2018 21:57:36 -0800 (PST) Received: from localhost ([::1]:47276 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj7n-0005ue-0X for importer@patchew.org; Mon, 05 Mar 2018 00:57:27 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:39326) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj63-0004zy-7n for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:40 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1esj5y-0008Em-W4 for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:39 -0500 Received: from mga06.intel.com ([134.134.136.31]:54637) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1esj5y-0008DY-Hk for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:34 -0500 Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 04 Mar 2018 21:55:33 -0800 Received: from yzhang13-kvm.sh.intel.com ([10.239.36.9]) by fmsmga005.fm.intel.com with ESMTP; 04 Mar 2018 21:55:32 -0800 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,426,1515484800"; d="scan'208";a="208826745" From: Yulei Zhang To: qemu-devel@nongnu.org Date: Mon, 5 Mar 2018 14:00:51 +0800 Message-Id: <1520229653-10658-3-git-send-email-yulei.zhang@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> References: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 134.134.136.31 Subject: [Qemu-devel] [PATCH V3 2/4] vfio: Add vm status change callback to stop/restart the mdev device X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Yulei Zhang , kevin.tian@intel.com, alex.williamson@redhat.com, kwankhede@nvidia.com, zhenyuw@linux.intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" VM status change handler is added to change the vfio pci device status during the migration, write the demanded device status to the DEVICE STATUS subregion to stop the device on the source side before fetch its status and start the deivce on the target side after restore its status. Signed-off-by: Yulei Zhang --- hw/vfio/pci.c | 20 ++++++++++++++++++++ include/hw/vfio/vfio-common.h | 1 + linux-headers/linux/vfio.h | 3 +++ 3 files changed, 24 insertions(+) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 2fe20e4..3e2289c 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -38,6 +38,7 @@ static void vfio_disable_interrupts(VFIOPCIDevice *vdev); static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled); static VMStateDescription vfio_pci_vmstate; +static void vfio_vm_change_state_handler(void *pv, int running, RunState s= tate); =20 /* * Disabling BAR mmaping can be slow, but toggling it around INTx can @@ -2880,6 +2881,7 @@ static void vfio_realize(PCIDevice *pdev, Error **err= p) vfio_register_err_notifier(vdev); vfio_register_req_notifier(vdev); vfio_setup_resetfn_quirk(vdev); + qemu_add_vm_change_state_handler(vfio_vm_change_state_handler, vdev); =20 return; =20 @@ -2962,6 +2964,24 @@ post_reset: vfio_pci_post_reset(vdev); } =20 +static void vfio_vm_change_state_handler(void *pv, int running, RunState s= tate) +{ + VFIOPCIDevice *vdev =3D pv; + VFIODevice *vbasedev =3D &vdev->vbasedev; + uint8_t dev_state; + uint8_t sz =3D 1; + + dev_state =3D running ? VFIO_DEVICE_START : VFIO_DEVICE_STOP; + + if (pwrite(vdev->vbasedev.fd, &dev_state, + sz, vdev->device_state.offset) !=3D sz) { + error_report("vfio: Failed to %s device", running ? "start" : "sto= p"); + return; + } + + vbasedev->device_state =3D dev_state; +} + static void vfio_instance_init(Object *obj) { PCIDevice *pci_dev =3D PCI_DEVICE(obj); diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index f3a2ac9..9c14a8f 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -125,6 +125,7 @@ typedef struct VFIODevice { unsigned int num_irqs; unsigned int num_regions; unsigned int flags; + bool device_state; } VFIODevice; =20 struct VFIODeviceOps { diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h index c3b8e4a..4ddeebc 100644 --- a/linux-headers/linux/vfio.h +++ b/linux-headers/linux/vfio.h @@ -303,6 +303,9 @@ struct vfio_region_info_cap_type { /* Mdev sub-type for device state save and restore */ #define VFIO_REGION_SUBTYPE_DEVICE_STATE (4) =20 +#define VFIO_DEVICE_START 0 +#define VFIO_DEVICE_STOP 1 + /** * VFIO_DEVICE_GET_IRQ_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 9, * struct vfio_irq_info) --=20 2.7.4 From nobody Fri May 17 00:57:11 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1520229570660238.3423771934797; Sun, 4 Mar 2018 21:59:30 -0800 (PST) Received: from localhost ([::1]:47284 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj9l-0007tC-T4 for importer@patchew.org; Mon, 05 Mar 2018 00:59:29 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:39350) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj64-000501-CZ for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:41 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1esj62-0008Fs-U6 for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:40 -0500 Received: from mga06.intel.com ([134.134.136.31]:54640) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1esj62-0008E6-GR for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:38 -0500 Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 04 Mar 2018 21:55:37 -0800 Received: from yzhang13-kvm.sh.intel.com ([10.239.36.9]) by fmsmga005.fm.intel.com with ESMTP; 04 Mar 2018 21:55:35 -0800 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,426,1515484800"; d="scan'208";a="208826760" From: Yulei Zhang To: qemu-devel@nongnu.org Date: Mon, 5 Mar 2018 14:00:52 +0800 Message-Id: <1520229653-10658-4-git-send-email-yulei.zhang@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> References: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 134.134.136.31 Subject: [Qemu-devel] [PATCH V3 3/4] vfio: Add struct vfio_vmstate_info to introduce put/get callback funtion for vfio device status save/restore X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Yulei Zhang , kevin.tian@intel.com, alex.williamson@redhat.com, kwankhede@nvidia.com, zhenyuw@linux.intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Introduce vfio_device_put/vfio_device_get funtion for vfio device state save/restore usage. For VFIO pci device status migrate, on the source side with funtion vfio_device_put to save the following states 1. pci configuration space addr0~addr5 2. pci configuration space msi_addr msi_data 3. pci device status fetch from device driver And on the target side with funtion vfio_device_get to restore the same states 1. re-setup the pci bar configuration 2. re-setup the pci device msi configuration 3. restore the pci device status Signed-off-by: Yulei Zhang --- hw/vfio/pci.c | 137 +++++++++++++++++++++++++++++++++++++++++= ++++ linux-headers/linux/vfio.h | 3 + 2 files changed, 140 insertions(+) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 3e2289c..c1676cf 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -2982,6 +2982,123 @@ static void vfio_vm_change_state_handler(void *pv, = int running, RunState state) vbasedev->device_state =3D dev_state; } =20 +static int vfio_device_put(QEMUFile *f, void *pv, size_t size, + VMStateField *field, QJSON *vmdesc) +{ + VFIOPCIDevice *vdev =3D pv; + PCIDevice *pdev =3D &vdev->pdev; + int sz =3D vdev->device_state.size - VFIO_DEVICE_STATE_OFFSET; + uint8_t *buf =3D NULL; + uint32_t msi_cfg, msi_lo, msi_hi, msi_data, bar_cfg, i; + bool msi_64bit; + + for (i =3D 0; i < PCI_ROM_SLOT; i++) { + bar_cfg =3D pci_default_read_config(pdev, PCI_BASE_ADDRESS_0 + i *= 4, 4); + qemu_put_be32(f, bar_cfg); + } + + msi_cfg =3D pci_default_read_config(pdev, pdev->msi_cap + PCI_MSI_FLAG= S, 2); + msi_64bit =3D !!(msi_cfg & PCI_MSI_FLAGS_64BIT); + + msi_lo =3D pci_default_read_config(pdev, + pdev->msi_cap + PCI_MSI_ADDRESS_LO, 4= ); + qemu_put_be32(f, msi_lo); + + if (msi_64bit) { + msi_hi =3D pci_default_read_config(pdev, + pdev->msi_cap + PCI_MSI_ADDRESS_H= I, + 4); + qemu_put_be32(f, msi_hi); + } + + msi_data =3D pci_default_read_config(pdev, + pdev->msi_cap + (msi_64bit ? PCI_MSI_DATA_64 : PCI_MSI_DATA= _32), + 2); + qemu_put_be32(f, msi_data); + + buf =3D g_malloc(sz); + if (buf =3D=3D NULL) { + error_report("vfio: Failed to allocate memory for migrate"); + goto exit; + } + + if (pread(vdev->vbasedev.fd, buf, sz, + vdev->device_state.offset + VFIO_DEVICE_STATE_OFFSET) !=3D s= z) { + error_report("vfio: Failed to read Device State Region"); + goto exit; + } + + qemu_put_buffer(f, buf, sz); + +exit: + g_free(buf); + + return 0; +} + +static int vfio_device_get(QEMUFile *f, void *pv, + size_t size, VMStateField *field) +{ + VFIOPCIDevice *vdev =3D pv; + PCIDevice *pdev =3D &vdev->pdev; + int sz =3D vdev->device_state.size - VFIO_DEVICE_STATE_OFFSET; + uint8_t *buf =3D NULL; + uint32_t ctl, msi_lo, msi_hi, msi_data, bar_cfg, i; + bool msi_64bit; + + /* retore pci bar configuration */ + ctl =3D pci_default_read_config(pdev, PCI_COMMAND, 2); + vfio_pci_write_config(pdev, PCI_COMMAND, + ctl & (!(PCI_COMMAND_IO | PCI_COMMAND_MEMORY)), = 2); + for (i =3D 0; i < PCI_ROM_SLOT; i++) { + bar_cfg =3D qemu_get_be32(f); + vfio_pci_write_config(pdev, PCI_BASE_ADDRESS_0 + i * 4, bar_cfg, 4= ); + } + vfio_pci_write_config(pdev, PCI_COMMAND, + ctl | PCI_COMMAND_IO | PCI_COMMAND_MEMORY, 2); + + /* restore msi configuration */ + ctl =3D pci_default_read_config(pdev, pdev->msi_cap + PCI_MSI_FLAGS, 2= ); + msi_64bit =3D !!(ctl & PCI_MSI_FLAGS_64BIT); + + vfio_pci_write_config(&vdev->pdev, + pdev->msi_cap + PCI_MSI_FLAGS, + ctl & (!PCI_MSI_FLAGS_ENABLE), 2); + + msi_lo =3D qemu_get_be32(f); + vfio_pci_write_config(pdev, pdev->msi_cap + PCI_MSI_ADDRESS_LO, msi_lo= , 4); + + if (msi_64bit) { + msi_hi =3D qemu_get_be32(f); + vfio_pci_write_config(pdev, pdev->msi_cap + PCI_MSI_ADDRESS_HI, + msi_hi, 4); + } + msi_data =3D qemu_get_be32(f); + vfio_pci_write_config(pdev, + pdev->msi_cap + (msi_64bit ? PCI_MSI_DATA_64 : PCI_MSI_DATA_= 32), + msi_data, 2); + + vfio_pci_write_config(&vdev->pdev, pdev->msi_cap + PCI_MSI_FLAGS, + ctl | PCI_MSI_FLAGS_ENABLE, 2); + + buf =3D g_malloc(sz); + if (buf =3D=3D NULL) { + error_report("vfio: Failed to allocate memory for migrate"); + return -1; + } + + qemu_get_buffer(f, buf, sz); + if (pwrite(vdev->vbasedev.fd, buf, sz, + vdev->device_state.offset + VFIO_DEVICE_STATE_OFFSET) !=3D = sz) { + error_report("vfio: Failed to write Device State Region"); + return -1; + } + + g_free(buf); + + return 0; +} + static void vfio_instance_init(Object *obj) { PCIDevice *pci_dev =3D PCI_DEVICE(obj); @@ -3026,9 +3143,29 @@ static Property vfio_pci_dev_properties[] =3D { DEFINE_PROP_END_OF_LIST(), }; =20 +static const VMStateInfo vfio_vmstate_info =3D { + .name =3D "vfio-state", + .get =3D vfio_device_get, + .put =3D vfio_device_put, +}; + static VMStateDescription vfio_pci_vmstate =3D { .name =3D "vfio-pci", .unmigratable =3D 1, + .version_id =3D 1, + .minimum_version_id =3D 1, + .fields =3D (VMStateField[]) { + { + .name =3D "vfio dev", + .version_id =3D 0, + .field_exists =3D NULL, + .size =3D 0, + .info =3D &vfio_vmstate_info, + .flags =3D VMS_SINGLE, + .offset =3D 0, + }, + VMSTATE_END_OF_LIST() + }, }; =20 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data) diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h index 4ddeebc..4451a8f 100644 --- a/linux-headers/linux/vfio.h +++ b/linux-headers/linux/vfio.h @@ -303,6 +303,9 @@ struct vfio_region_info_cap_type { /* Mdev sub-type for device state save and restore */ #define VFIO_REGION_SUBTYPE_DEVICE_STATE (4) =20 +/* Offset in region to save device state */ +#define VFIO_DEVICE_STATE_OFFSET 1 + #define VFIO_DEVICE_START 0 #define VFIO_DEVICE_STOP 1 =20 --=20 2.7.4 From nobody Fri May 17 00:57:11 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 152022957036710.837653128916259; Sun, 4 Mar 2018 21:59:30 -0800 (PST) Received: from localhost ([::1]:47283 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj9l-0007t7-Lm for importer@patchew.org; Mon, 05 Mar 2018 00:59:29 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:39361) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj67-00051m-3m for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:46 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1esj66-0008Gq-5A for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:43 -0500 Received: from mga06.intel.com ([134.134.136.31]:54640) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1esj65-0008E6-SW for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:42 -0500 Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 04 Mar 2018 21:55:41 -0800 Received: from yzhang13-kvm.sh.intel.com ([10.239.36.9]) by fmsmga005.fm.intel.com with ESMTP; 04 Mar 2018 21:55:40 -0800 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,426,1515484800"; d="scan'208";a="208826774" From: Yulei Zhang To: qemu-devel@nongnu.org Date: Mon, 5 Mar 2018 14:00:53 +0800 Message-Id: <1520229653-10658-5-git-send-email-yulei.zhang@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> References: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 134.134.136.31 Subject: [Qemu-devel] [PATCH V3 4/4] vifo: introduce new VFIO ioctl VFIO_IOMMU_GET_DIRTY_BITMAP X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Yulei Zhang , kevin.tian@intel.com, alex.williamson@redhat.com, kwankhede@nvidia.com, zhenyuw@linux.intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" New VFIO ioctl VFIO_IOMMU_GET_DIRTY_BITMAP is used to fetch the bitmap of pinned memory in iommu container, we need copy those memory to the target during the migration as they are dirtied by mdev devices. Signed-off-by: Yulei Zhang --- hw/vfio/common.c | 34 ++++++++++++++++++++++++++++++++++ linux-headers/linux/vfio.h | 14 ++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 7b2924c..a952554 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -35,6 +35,7 @@ #include "sysemu/kvm.h" #include "trace.h" #include "qapi/error.h" +#include "exec/ram_addr.h" =20 struct vfio_group_head vfio_group_list =3D QLIST_HEAD_INITIALIZER(vfio_group_list); @@ -624,9 +625,42 @@ static void vfio_listener_region_del(MemoryListener *l= istener, } } =20 +static void vfio_log_sync(MemoryListener *listener, + MemoryRegionSection *section) +{ + VFIOContainer *container =3D container_of(listener, VFIOContainer, lis= tener); + VFIOGroup *group =3D QLIST_FIRST(&container->group_list); + VFIODevice *vbasedev; + QLIST_FOREACH(vbasedev, &group->device_list, next) { + if (vbasedev->device_state =3D=3D VFIO_DEVICE_START) { + return; + } + } + + struct vfio_iommu_get_dirty_bitmap *d; + ram_addr_t size =3D int128_get64(section->size); + unsigned long page_nr =3D size >> TARGET_PAGE_BITS; + unsigned long bitmap_size =3D + (BITS_TO_LONGS(page_nr) + 1) * sizeof(unsigned long); + d =3D g_malloc0(sizeof(*d) + bitmap_size); + d->start_addr =3D section->offset_within_address_space; + d->page_nr =3D page_nr; + + if (ioctl(container->fd, VFIO_IOMMU_GET_DIRTY_BITMAP, d)) { + error_report("vfio: Failed to fetch dirty pages for migration"); + goto exit; + } + + cpu_physical_memory_set_dirty_lebitmap((unsigned long *)&d->dirty_bitm= ap, + d->start_addr, d->page_nr); +exit: + g_free(d); +} + static const MemoryListener vfio_memory_listener =3D { .region_add =3D vfio_listener_region_add, .region_del =3D vfio_listener_region_del, + .log_sync =3D vfio_log_sync, }; =20 static void vfio_listener_release(VFIOContainer *container) diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h index 4451a8f..a41f73b 100644 --- a/linux-headers/linux/vfio.h +++ b/linux-headers/linux/vfio.h @@ -574,6 +574,20 @@ struct vfio_iommu_type1_dma_unmap { #define VFIO_IOMMU_ENABLE _IO(VFIO_TYPE, VFIO_BASE + 15) #define VFIO_IOMMU_DISABLE _IO(VFIO_TYPE, VFIO_BASE + 16) =20 +/** + * VFIO_IOMMU_GET_DIRTY_BITMAP - _IOW(VFIO_TYPE, VFIO_BASE + 17, + * struct vfio_iommu_get_dirty_bitmap) + * + * Return: 0 on success, -errno on failure. + */ +struct vfio_iommu_get_dirty_bitmap { + __u64 start_addr; + __u64 page_nr; + __u8 dirty_bitmap[]; +}; + +#define VFIO_IOMMU_GET_DIRTY_BITMAP _IO(VFIO_TYPE, VFIO_BASE + 17) + /* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */ =20 /* --=20 2.7.4