Implement zPCI device state migration, consequently enabling migration
of VMs that have emulated PCI devices, whether virtio or not.
Migration is allowed for devices whose function handle has the
FH_SHM_EMUL bit set. For these devices QEMU will save and restore the
state of its zPCI emulator.
Passthrough devices will continue to block migration.
Signed-off-by: Konstantin Shkolnyy <kshk@linux.ibm.com>
---
hw/s390x/s390-pci-bus.c | 139 ++++++++++++++++++++++++++++++--
hw/s390x/s390-pci-inst.c | 2 +-
hw/s390x/s390-virtio-ccw.c | 5 ++
include/hw/s390x/s390-pci-bus.h | 4 +
4 files changed, 144 insertions(+), 6 deletions(-)
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index f339a0fd94..ab9339bdc7 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -26,6 +26,7 @@
#include "hw/pci/pci_bridge.h"
#include "hw/pci/msi.h"
#include "exec/cpu-common.h"
+#include "migration/blocker.h"
#include "qemu/error-report.h"
#include "qemu/module.h"
#include "system/reset.h"
@@ -33,6 +34,11 @@
#include "trace.h"
+static const Property phb_props[] = {
+ DEFINE_PROP_BOOL("x-zpci-emul-dev-migr-enabled", S390pciState,
+ emul_dev_migr_enabled, true),
+};
+
S390pciState *s390_get_phb(void)
{
static S390pciState *phb;
@@ -904,6 +910,25 @@ static void set_pbdev_info(S390PCIBusDevice *pbdev)
pbdev->pci_group = s390_group_find(ZPCI_DEFAULT_FN_GRP);
}
+static int s390_set_emul_dev_migration_blocker(S390pciState *s, Error **errp)
+{
+ if (s->emul_dev_migr_enabled) {
+ return 0;
+ }
+ error_setg(&s->migr_blocker,
+ "Migration disabled for emulated zPCI devices on this machine type");
+ return migrate_add_blocker(&s->migr_blocker, errp);
+}
+
+static void s390_clear_emul_dev_migration_blocker(S390pciState *s)
+{
+ if (s->migr_blocker) {
+ migrate_del_blocker(&s->migr_blocker);
+ error_free(s->migr_blocker);
+ s->migr_blocker = NULL;
+ }
+}
+
static void s390_pcihost_realize(DeviceState *dev, Error **errp)
{
PCIBus *b;
@@ -939,6 +964,7 @@ static void s390_pcihost_realize(DeviceState *dev, Error **errp)
css_register_io_adapters(CSS_IO_ADAPTER_PCI, true, false,
S390_ADAPTER_SUPPRESSIBLE, errp);
s390_pcihost_kvm_realize();
+ s390_set_emul_dev_migration_blocker(s, errp);
}
static void s390_pcihost_unrealize(DeviceState *dev)
@@ -946,6 +972,8 @@ static void s390_pcihost_unrealize(DeviceState *dev)
S390PCIGroup *group;
S390pciState *s = S390_PCI_HOST_BRIDGE(dev);
+ s390_clear_emul_dev_migration_blocker(s);
+
while (!QTAILQ_EMPTY(&s->zpci_groups)) {
group = QTAILQ_FIRST(&s->zpci_groups);
QTAILQ_REMOVE(&s->zpci_groups, group, link);
@@ -1116,6 +1144,27 @@ static int s390_pci_interp_plug(S390pciState *s, S390PCIBusDevice *pbdev)
return 0;
}
+static int s390_set_passthrough_migration_blocker(S390PCIBusDevice *pbdev,
+ Error **errp)
+{
+ pbdev->passthrough_migr_blocker = NULL;
+
+ if (pbdev->fh & FH_SHM_EMUL) {
+ return 0;
+ }
+ error_setg(&pbdev->passthrough_migr_blocker,
+ "Migration blocked by passthrough zPCI device "
+ "fh 0x%x uid %d fid %d", pbdev->fh, pbdev->uid, pbdev->fid);
+ return migrate_add_blocker(&pbdev->passthrough_migr_blocker, errp);
+}
+
+static void s390_clear_passthrough_migration_blocker(S390PCIBusDevice *pbdev)
+{
+ if (pbdev->passthrough_migr_blocker) {
+ migrate_del_blocker(&pbdev->passthrough_migr_blocker);
+ }
+}
+
static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
@@ -1235,6 +1284,11 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
return;
}
+ if (s390_set_passthrough_migration_blocker(pbdev, errp)) {
+ s390_pci_msix_free(pbdev);
+ return;
+ }
+
if (dev->hotplugged) {
s390_pci_generate_plug_event(HP_EVENT_TO_CONFIGURED ,
pbdev->fh, pbdev->fid);
@@ -1271,6 +1325,8 @@ static void s390_pcihost_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
return;
}
+ s390_clear_passthrough_migration_blocker(pbdev);
+
s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
pbdev->fh, pbdev->fid);
bus = pci_get_bus(pci_dev);
@@ -1438,6 +1494,7 @@ static void s390_pcihost_class_init(ObjectClass *klass, const void *data)
hc->unplug_request = s390_pcihost_unplug_request;
hc->unplug = s390_pcihost_unplug;
msi_nonbroken = true;
+ device_class_set_props(dc, phb_props);
}
static const TypeInfo s390_pcihost_info = {
@@ -1600,13 +1657,85 @@ static const Property s390_pci_device_properties[] = {
true),
};
-static const VMStateDescription s390_pci_device_vmstate = {
- .name = TYPE_S390_PCI_DEVICE,
+static int s390_pci_device_post_load(void *opaque, int version_id)
+{
+ S390PCIBusDevice *pbdev = S390_PCI_DEVICE(opaque);
+
/*
- * TODO: add state handling here, so migration works at least with
- * emulated pci devices on s390x
+ * Now that S390PCIBusDevice fields have been restored, regenerate IOMMU
+ * state - that includes IOTLB contents and QEMU memory regions.
*/
- .unmigratable = 1,
+ if (pbdev->iommu_enabled) {
+ assert(pbdev->iommu);
+ if (s390_pci_is_translation_enabled(pbdev->g_iota)) {
+ s390_pci_iommu_enable(pbdev);
+ s390_pci_ioat_replay(pbdev);
+ } else {
+ s390_pci_iommu_direct_map_enable(pbdev);
+ }
+ }
+
+ /*
+ * Guest sets fmb_addr by mpcifc.ZPCI_MOD_FC_SET_MEASURE instruction.
+ * The handler consequently starts fmb_timer. Now that fmb_addr has been
+ * restored, we may need to restart fmb_timer.
+ */
+ if (pbdev->fmb_addr) {
+ assert(!pbdev->fmb_timer);
+ assert(pbdev->pci_group);
+ pbdev->fmb_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+ fmb_update, pbdev);
+ timer_mod(pbdev->fmb_timer,
+ qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+ pbdev->pci_group->zpci_group.mui);
+ }
+ return 0;
+}
+
+static const VMStateDescription s390_pci_device_vmstate = {
+ .name = TYPE_S390_PCI_DEVICE,
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .post_load = s390_pci_device_post_load,
+ .fields = (const VMStateField[]) {
+ VMSTATE_UINT32(state, S390PCIBusDevice),
+ VMSTATE_UINT16(uid, S390PCIBusDevice),
+ VMSTATE_UINT32(idx, S390PCIBusDevice),
+ VMSTATE_UINT32(fh, S390PCIBusDevice),
+ VMSTATE_UINT32(fid, S390PCIBusDevice),
+ VMSTATE_BOOL(fid_defined, S390PCIBusDevice),
+ VMSTATE_UINT64(fmb_addr, S390PCIBusDevice),
+ VMSTATE_UINT32(fmb.format, S390PCIBusDevice),
+ VMSTATE_UINT32(fmb.sample, S390PCIBusDevice),
+ VMSTATE_UINT64(fmb.last_update, S390PCIBusDevice),
+ VMSTATE_UINT64_ARRAY(fmb.counter, S390PCIBusDevice,
+ ARRAY_SIZE(((S390PCIBusDevice *)0)->fmb.counter)),
+ VMSTATE_UINT64(fmb.fmt0.dma_rbytes, S390PCIBusDevice),
+ VMSTATE_UINT64(fmb.fmt0.dma_wbytes, S390PCIBusDevice),
+ VMSTATE_UINT8(isc, S390PCIBusDevice),
+ VMSTATE_UINT16(noi, S390PCIBusDevice),
+ VMSTATE_UINT8(sum, S390PCIBusDevice),
+ VMSTATE_UINT8(pft, S390PCIBusDevice),
+ VMSTATE_UINT64(routes.adapter.ind_addr, S390PCIBusDevice),
+ VMSTATE_UINT64(routes.adapter.summary_addr, S390PCIBusDevice),
+ VMSTATE_UINT64(routes.adapter.ind_offset, S390PCIBusDevice),
+ VMSTATE_UINT32(routes.adapter.summary_offset, S390PCIBusDevice),
+ VMSTATE_UINT32(routes.adapter.adapter_id, S390PCIBusDevice),
+ VMSTATE_BOOL(iommu_enabled, S390PCIBusDevice),
+ VMSTATE_UINT64(g_iota, S390PCIBusDevice),
+ VMSTATE_UINT64(pba, S390PCIBusDevice),
+ VMSTATE_UINT64(pal, S390PCIBusDevice),
+ VMSTATE_UINT64(max_dma_limit, S390PCIBusDevice),
+ VMSTATE_PTR_TO_IND_ADDR(summary_ind, S390PCIBusDevice),
+ VMSTATE_PTR_TO_IND_ADDR(indicator, S390PCIBusDevice),
+ VMSTATE_BOOL(pci_unplug_request_processed, S390PCIBusDevice),
+ VMSTATE_BOOL(unplug_requested, S390PCIBusDevice),
+ VMSTATE_BOOL(interp, S390PCIBusDevice),
+ VMSTATE_BOOL(forwarding_assist, S390PCIBusDevice),
+ VMSTATE_BOOL(aif, S390PCIBusDevice),
+ VMSTATE_BOOL(rtr_avail, S390PCIBusDevice),
+ VMSTATE_END_OF_LIST()
+ }
};
static void s390_pci_device_class_init(ObjectClass *klass, const void *data)
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 191f549ba7..a9c0a4effb 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -1094,7 +1094,7 @@ static int fmb_do_update(S390PCIBusDevice *pbdev, int offset, uint64_t val,
return ret;
}
-static void fmb_update(void *opaque)
+void fmb_update(void *opaque)
{
S390PCIBusDevice *pbdev = opaque;
int64_t t = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 3ef009463d..da48a2edc0 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -913,6 +913,11 @@ static void ccw_machine_11_0_instance_options(MachineState *machine)
static void ccw_machine_11_0_class_options(MachineClass *mc)
{
+ static GlobalProperty compat[] = {
+ { TYPE_S390_PCI_HOST_BRIDGE, "x-zpci-emul-dev-migr-enabled", "off" },
+ };
+
+ compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
}
DEFINE_CCW_MACHINE_AS_LATEST(11, 0);
diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h
index 2fab28e6e0..15cf004c31 100644
--- a/include/hw/s390x/s390-pci-bus.h
+++ b/include/hw/s390x/s390-pci-bus.h
@@ -337,6 +337,7 @@ struct S390PCIBusDevice {
uint16_t uid;
uint32_t idx;
uint32_t fh;
+ Error *passthrough_migr_blocker;
uint32_t fid;
bool fid_defined;
uint64_t fmb_addr;
@@ -390,6 +391,8 @@ struct S390pciState {
QTAILQ_HEAD(, S390PCIDMACount) zpci_dma_limit;
QTAILQ_HEAD(, S390PCIGroup) zpci_groups;
uint8_t next_sim_grp;
+ bool emul_dev_migr_enabled;
+ Error *migr_blocker;
};
S390pciState *s390_get_phb(void);
@@ -415,5 +418,6 @@ S390PCIBusDevice *s390_pci_find_dev_by_pci(S390pciState *s,
S390PCIBusDevice *s390_pci_find_next_avail_dev(S390pciState *s,
S390PCIBusDevice *pbdev);
void s390_pci_ism_reset(void);
+void fmb_update(void *opaque);
#endif
--
2.34.1