From nobody Tue Feb 10 12:57:39 2026 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1520229570660238.3423771934797; Sun, 4 Mar 2018 21:59:30 -0800 (PST) Received: from localhost ([::1]:47284 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj9l-0007tC-T4 for importer@patchew.org; Mon, 05 Mar 2018 00:59:29 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:39350) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1esj64-000501-CZ for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:41 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1esj62-0008Fs-U6 for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:40 -0500 Received: from mga06.intel.com ([134.134.136.31]:54640) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1esj62-0008E6-GR for qemu-devel@nongnu.org; Mon, 05 Mar 2018 00:55:38 -0500 Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 04 Mar 2018 21:55:37 -0800 Received: from yzhang13-kvm.sh.intel.com ([10.239.36.9]) by fmsmga005.fm.intel.com with ESMTP; 04 Mar 2018 21:55:35 -0800 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,426,1515484800"; d="scan'208";a="208826760" From: Yulei Zhang To: qemu-devel@nongnu.org Date: Mon, 5 Mar 2018 14:00:52 +0800 Message-Id: <1520229653-10658-4-git-send-email-yulei.zhang@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> References: <1520229653-10658-1-git-send-email-yulei.zhang@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 134.134.136.31 Subject: [Qemu-devel] [PATCH V3 3/4] vfio: Add struct vfio_vmstate_info to introduce put/get callback funtion for vfio device status save/restore X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Yulei Zhang , kevin.tian@intel.com, alex.williamson@redhat.com, kwankhede@nvidia.com, zhenyuw@linux.intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Introduce vfio_device_put/vfio_device_get funtion for vfio device state save/restore usage. For VFIO pci device status migrate, on the source side with funtion vfio_device_put to save the following states 1. pci configuration space addr0~addr5 2. pci configuration space msi_addr msi_data 3. pci device status fetch from device driver And on the target side with funtion vfio_device_get to restore the same states 1. re-setup the pci bar configuration 2. re-setup the pci device msi configuration 3. restore the pci device status Signed-off-by: Yulei Zhang --- hw/vfio/pci.c | 137 +++++++++++++++++++++++++++++++++++++++++= ++++ linux-headers/linux/vfio.h | 3 + 2 files changed, 140 insertions(+) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 3e2289c..c1676cf 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -2982,6 +2982,123 @@ static void vfio_vm_change_state_handler(void *pv, = int running, RunState state) vbasedev->device_state =3D dev_state; } =20 +static int vfio_device_put(QEMUFile *f, void *pv, size_t size, + VMStateField *field, QJSON *vmdesc) +{ + VFIOPCIDevice *vdev =3D pv; + PCIDevice *pdev =3D &vdev->pdev; + int sz =3D vdev->device_state.size - VFIO_DEVICE_STATE_OFFSET; + uint8_t *buf =3D NULL; + uint32_t msi_cfg, msi_lo, msi_hi, msi_data, bar_cfg, i; + bool msi_64bit; + + for (i =3D 0; i < PCI_ROM_SLOT; i++) { + bar_cfg =3D pci_default_read_config(pdev, PCI_BASE_ADDRESS_0 + i *= 4, 4); + qemu_put_be32(f, bar_cfg); + } + + msi_cfg =3D pci_default_read_config(pdev, pdev->msi_cap + PCI_MSI_FLAG= S, 2); + msi_64bit =3D !!(msi_cfg & PCI_MSI_FLAGS_64BIT); + + msi_lo =3D pci_default_read_config(pdev, + pdev->msi_cap + PCI_MSI_ADDRESS_LO, 4= ); + qemu_put_be32(f, msi_lo); + + if (msi_64bit) { + msi_hi =3D pci_default_read_config(pdev, + pdev->msi_cap + PCI_MSI_ADDRESS_H= I, + 4); + qemu_put_be32(f, msi_hi); + } + + msi_data =3D pci_default_read_config(pdev, + pdev->msi_cap + (msi_64bit ? PCI_MSI_DATA_64 : PCI_MSI_DATA= _32), + 2); + qemu_put_be32(f, msi_data); + + buf =3D g_malloc(sz); + if (buf =3D=3D NULL) { + error_report("vfio: Failed to allocate memory for migrate"); + goto exit; + } + + if (pread(vdev->vbasedev.fd, buf, sz, + vdev->device_state.offset + VFIO_DEVICE_STATE_OFFSET) !=3D s= z) { + error_report("vfio: Failed to read Device State Region"); + goto exit; + } + + qemu_put_buffer(f, buf, sz); + +exit: + g_free(buf); + + return 0; +} + +static int vfio_device_get(QEMUFile *f, void *pv, + size_t size, VMStateField *field) +{ + VFIOPCIDevice *vdev =3D pv; + PCIDevice *pdev =3D &vdev->pdev; + int sz =3D vdev->device_state.size - VFIO_DEVICE_STATE_OFFSET; + uint8_t *buf =3D NULL; + uint32_t ctl, msi_lo, msi_hi, msi_data, bar_cfg, i; + bool msi_64bit; + + /* retore pci bar configuration */ + ctl =3D pci_default_read_config(pdev, PCI_COMMAND, 2); + vfio_pci_write_config(pdev, PCI_COMMAND, + ctl & (!(PCI_COMMAND_IO | PCI_COMMAND_MEMORY)), = 2); + for (i =3D 0; i < PCI_ROM_SLOT; i++) { + bar_cfg =3D qemu_get_be32(f); + vfio_pci_write_config(pdev, PCI_BASE_ADDRESS_0 + i * 4, bar_cfg, 4= ); + } + vfio_pci_write_config(pdev, PCI_COMMAND, + ctl | PCI_COMMAND_IO | PCI_COMMAND_MEMORY, 2); + + /* restore msi configuration */ + ctl =3D pci_default_read_config(pdev, pdev->msi_cap + PCI_MSI_FLAGS, 2= ); + msi_64bit =3D !!(ctl & PCI_MSI_FLAGS_64BIT); + + vfio_pci_write_config(&vdev->pdev, + pdev->msi_cap + PCI_MSI_FLAGS, + ctl & (!PCI_MSI_FLAGS_ENABLE), 2); + + msi_lo =3D qemu_get_be32(f); + vfio_pci_write_config(pdev, pdev->msi_cap + PCI_MSI_ADDRESS_LO, msi_lo= , 4); + + if (msi_64bit) { + msi_hi =3D qemu_get_be32(f); + vfio_pci_write_config(pdev, pdev->msi_cap + PCI_MSI_ADDRESS_HI, + msi_hi, 4); + } + msi_data =3D qemu_get_be32(f); + vfio_pci_write_config(pdev, + pdev->msi_cap + (msi_64bit ? PCI_MSI_DATA_64 : PCI_MSI_DATA_= 32), + msi_data, 2); + + vfio_pci_write_config(&vdev->pdev, pdev->msi_cap + PCI_MSI_FLAGS, + ctl | PCI_MSI_FLAGS_ENABLE, 2); + + buf =3D g_malloc(sz); + if (buf =3D=3D NULL) { + error_report("vfio: Failed to allocate memory for migrate"); + return -1; + } + + qemu_get_buffer(f, buf, sz); + if (pwrite(vdev->vbasedev.fd, buf, sz, + vdev->device_state.offset + VFIO_DEVICE_STATE_OFFSET) !=3D = sz) { + error_report("vfio: Failed to write Device State Region"); + return -1; + } + + g_free(buf); + + return 0; +} + static void vfio_instance_init(Object *obj) { PCIDevice *pci_dev =3D PCI_DEVICE(obj); @@ -3026,9 +3143,29 @@ static Property vfio_pci_dev_properties[] =3D { DEFINE_PROP_END_OF_LIST(), }; =20 +static const VMStateInfo vfio_vmstate_info =3D { + .name =3D "vfio-state", + .get =3D vfio_device_get, + .put =3D vfio_device_put, +}; + static VMStateDescription vfio_pci_vmstate =3D { .name =3D "vfio-pci", .unmigratable =3D 1, + .version_id =3D 1, + .minimum_version_id =3D 1, + .fields =3D (VMStateField[]) { + { + .name =3D "vfio dev", + .version_id =3D 0, + .field_exists =3D NULL, + .size =3D 0, + .info =3D &vfio_vmstate_info, + .flags =3D VMS_SINGLE, + .offset =3D 0, + }, + VMSTATE_END_OF_LIST() + }, }; =20 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data) diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h index 4ddeebc..4451a8f 100644 --- a/linux-headers/linux/vfio.h +++ b/linux-headers/linux/vfio.h @@ -303,6 +303,9 @@ struct vfio_region_info_cap_type { /* Mdev sub-type for device state save and restore */ #define VFIO_REGION_SUBTYPE_DEVICE_STATE (4) =20 +/* Offset in region to save device state */ +#define VFIO_DEVICE_STATE_OFFSET 1 + #define VFIO_DEVICE_START 0 #define VFIO_DEVICE_STOP 1 =20 --=20 2.7.4