Support set the CPU affinity of the live migration thread to improve the migration performance in specific cases. By default, the migration thread shares CPU resources with the VM process. With those API, support pin migration thread to expected CPU list to avoid preempting CPU resources of VM process. New API 'pinMigrationThread' and virsh command 'virsh migrationpin' is used to pin migration thread to expected CPU list before or during migration. New migration param 'migration.pin' is used to support migration pin via interface 'virDomainMigrateToURI3'. Jiang Jiacheng (2): migration/migration-pin: support migration thread pin by virsh command migration/migration-pin/multifd-pin: add migration pin status handle zhengchuan (10): migration/migration-pin: get migration pid for migration pin migration/migration-pin: pin migration pid by given cpumap migration/migration-pin: add qemu monitor callback functions migration/migration-pin: add migrationpin for migration parameters migration/migration-pin: get cpumap from migration.pin migration/migration-pin: add domainMigrationPid for qemuMonitorCallbacks migration/multifd-pin: get multifd pid for migration pin migration/multifd-pin: pin multifd pid by given cpumap migration/multifd-pin: add qemu monitor callback functions migration/multifd-pin: support migration multifd thread pin include/libvirt/libvirt-domain.h | 19 ++ src/conf/domain_conf.c | 9 + src/conf/domain_conf.h | 11 ++ src/conf/virconftypes.h | 2 + src/driver-hypervisor.h | 16 ++ src/libvirt-domain.c | 144 +++++++++++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 7 + src/qemu/qemu_domain.c | 5 + src/qemu/qemu_domain.h | 5 + src/qemu/qemu_driver.c | 175 ++++++++++++++++++ src/qemu/qemu_migration.c | 5 + src/qemu/qemu_migration.h | 1 + src/qemu/qemu_migration_params.c | 21 +++ src/qemu/qemu_migration_params.h | 4 + src/qemu/qemu_monitor.c | 20 ++ src/qemu/qemu_monitor.h | 13 ++ src/qemu/qemu_monitor_json.c | 32 ++++ src/qemu/qemu_process.c | 303 +++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 15 ++ src/remote/remote_driver.c | 3 + src/remote/remote_protocol.x | 43 ++++- src/remote_protocol-structs | 28 +++ src/util/vircgroup.c | 3 + src/util/vircgroup.h | 1 + tools/virsh-domain.c | 69 +++++++ 26 files changed, 954 insertions(+), 1 deletion(-) -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> Firstly, we need to get migration pids, add virDomainMigrateGetMigrationPids() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 3 +++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 39 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 ++++ src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 31 +++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 8 +++++++ 10 files changed, 112 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index XXXXXXX..XXXXXXX 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -XXX,XX +XXX,XX @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, int seconds, unsigned int flags); +char *virDomainMigrateGetMigrationPids(virDomainPtr domain, + unsigned int flags); + #endif /* LIBVIRT_DOMAIN_H */ diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index XXXXXXX..XXXXXXX 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -XXX,XX +XXX,XX @@ typedef int int seconds, unsigned int flags); +typedef char * +(*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, + unsigned int flags); + typedef struct _virHypervisorDriver virHypervisorDriver; /** @@ -XXX,XX +XXX,XX @@ struct _virHypervisorDriver { virDrvDomainAuthorizedSSHKeysSet domainAuthorizedSSHKeysSet; virDrvDomainGetMessages domainGetMessages; virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; + virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -XXX,XX +XXX,XX @@ virDomainStartDirtyRateCalc(virDomainPtr domain, virDispatchError(conn); return -1; } + +/** + * virDomainMigrateGetMigrationPids: + * @domain: pointer to domain object + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Get migration thread pid. + * + * Returns the migration pids which must be freed by the caller, or + * NULL if there was an error. + * + * Since: 9.1.0 + */ +char * +virDomainMigrateGetMigrationPids(virDomainPtr domain, + unsigned int flags) +{ + virConnectPtr conn; + VIR_DOMAIN_DEBUG(domain, "migration pids flags=0x%x", flags); + virResetLastError(); + virCheckDomainReturn(domain, NULL); + conn = domain->conn; + + virCheckReadOnlyGoto(domain->conn->flags, error); + + if (conn->driver->domainMigrateGetMigrationPids) { + char *ret; + ret = conn->driver->domainMigrateGetMigrationPids(domain, flags); + if (!ret) + goto error; + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return NULL; +} diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -XXX,XX +XXX,XX @@ LIBVIRT_8.5.0 { virDomainAbortJobFlags; } LIBVIRT_8.4.0; +LIBVIRT_9.1.0 { + global: + virDomainMigrateGetMigrationPids; +} LIBVIRT_8.5.0; + # .... define new API here using predicted next version number .... diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -XXX,XX +XXX,XX @@ qemuDomainObjPrivateDataClear(qemuDomainObjPrivate *priv) /* remove automatic pinning data */ g_clear_pointer(&priv->autoNodeset, virBitmapFree); g_clear_pointer(&priv->autoCpuset, virBitmapFree); + g_clear_pointer(&priv->pcpumap, virBitmapFree); g_clear_pointer(&priv->pciaddrs, virDomainPCIAddressSetFree); g_clear_pointer(&priv->usbaddrs, virDomainUSBAddressSetFree); g_clear_pointer(&priv->origCPU, virCPUDefFree); @@ -XXX,XX +XXX,XX @@ qemuDomainObjPrivateFree(void *data) virObjectUnref(priv->monConfig); g_free(priv->lockState); g_free(priv->origname); + g_free(priv->migrationPids); virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -XXX,XX +XXX,XX @@ struct _qemuDomainObjPrivate { char *origname; int nbdPort; /* Port used for migration with NBD */ unsigned short migrationPort; + char *migrationPids; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case it was changed for the current @@ -XXX,XX +XXX,XX @@ struct _qemuDomainObjPrivate { /* Bitmaps below hold data from the auto NUMA feature */ virBitmap *autoNodeset; virBitmap *autoCpuset; + virBitmap *pcpumap; bool signalIOError; /* true if the domain condition should be signalled on I/O error */ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -XXX,XX +XXX,XX @@ qemuDomainStartDirtyRateCalc(virDomainPtr dom, return ret; } +static char * +qemuDomainMigrateGetMigrationPids(virDomainPtr dom, + unsigned int flags) +{ + char *ret = NULL; + virDomainObj *vm = NULL; + qemuDomainObjPrivate *priv = NULL; + + virCheckFlags(0, NULL); + + vm = qemuDomainObjFromDomain(dom); + if (!vm) + goto cleanup; + + if (virDomainMigrateGetMigrationPidsEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + priv = vm->privateData; + + if (priv->migrationPids) { + ret = g_strdup(priv->migrationPids); + if (!ret) + VIR_ERROR(_("failed to strdup migrationPids")); + } + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -XXX,XX +XXX,XX @@ static virHypervisorDriver qemuHypervisorDriver = { .domainGetMessages = qemuDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -XXX,XX +XXX,XX @@ static virHypervisorDriver hypervisor_driver = { .domainGetMessages = remoteDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index XXXXXXX..XXXXXXX 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -XXX,XX +XXX,XX @@ struct remote_domain_migrate_set_compression_cache_args { unsigned int flags; }; +struct remote_domain_migrate_get_migration_pids_args { + remote_nonnull_domain dom; + unsigned int flags; +}; + +struct remote_domain_migrate_get_migration_pids_ret { + remote_nonnull_string migrationPids; +}; + struct remote_domain_migrate_set_max_speed_args { remote_nonnull_domain dom; unsigned hyper bandwidth; @@ -XXX,XX +XXX,XX @@ enum remote_procedure { * @generate: both * @acl: domain:write */ - REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442 + REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index XXXXXXX..XXXXXXX 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -XXX,XX +XXX,XX @@ struct remote_domain_event_memory_device_size_change_msg { remote_nonnull_string alias; uint64_t size; }; +struct remote_domain_migrate_get_migration_pids_args { + remote_nonnull_domain dom; + u_int flags; +}; +struct remote_domain_migrate_get_migration_pids_ret { + remote_nonnull_string migrationPids; +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -XXX,XX +XXX,XX @@ enum remote_procedure { REMOTE_PROC_DOMAIN_SAVE_PARAMS = 440, REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441, REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, }; -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> Secondly, we start to pin migration pid by given cpumap, add virDomainPinMigrationThread() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 4 ++ src/conf/domain_conf.c | 9 ++++ src/conf/domain_conf.h | 11 ++++ src/conf/virconftypes.h | 2 + src/driver-hypervisor.h | 6 +++ src/libvirt-domain.c | 65 +++++++++++++++++++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 1 + src/qemu/qemu_driver.c | 91 ++++++++++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 ++++- src/remote_protocol-structs | 5 ++ src/util/vircgroup.c | 3 ++ src/util/vircgroup.h | 1 + 14 files changed, 212 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index XXXXXXX..XXXXXXX 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -XXX,XX +XXX,XX @@ int virDomainPinEmulator (virDomainPtr domain, int maplen, unsigned int flags); +int virDomainPinMigrationThread (virDomainPtr domain, + unsigned char *cpumap, + int maplen); + int virDomainGetEmulatorPinInfo (virDomainPtr domain, unsigned char *cpumaps, int maplen, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -XXX,XX +XXX,XX @@ virDomainDefHasSpiceGraphics(const virDomainDef *def) return false; } + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def) +{ + if (!def) + return; + virBitmapFree(def->cpumask); + VIR_FREE(def); +} diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -XXX,XX +XXX,XX @@ #include <libxml/xpath.h> #include "internal.h" +#include "viralloc.h" #include "virconftypes.h" #include "capabilities.h" #include "cpu_conf.h" @@ -XXX,XX +XXX,XX @@ virDomainObjGetMessages(virDomainObj *vm, bool virDomainDefHasSpiceGraphics(const virDomainDef *def); + +struct _virDomainMigrationIDDef { + bool autofill; + int thread_id; + virBitmap *cpumask; + virDomainThreadSchedParam sched; +}; + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def); diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h index XXXXXXX..XXXXXXX 100644 --- a/src/conf/virconftypes.h +++ b/src/conf/virconftypes.h @@ -XXX,XX +XXX,XX @@ typedef struct _virDomainXMLOption virDomainXMLOption; typedef struct _virDomainXMLPrivateDataCallbacks virDomainXMLPrivateDataCallbacks; typedef struct _virDomainXenbusControllerOpts virDomainXenbusControllerOpts; + +typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef; diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index XXXXXXX..XXXXXXX 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -XXX,XX +XXX,XX @@ typedef int int seconds, unsigned int flags); +typedef int +(*virDrvDomainPinMigrationThread)(virDomainPtr domain, + unsigned char *cpumap, + int maplen); + typedef char * (*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, unsigned int flags); @@ -XXX,XX +XXX,XX @@ struct _virHypervisorDriver { virDrvDomainAuthorizedSSHKeysSet domainAuthorizedSSHKeysSet; virDrvDomainGetMessages domainGetMessages; virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; + virDrvDomainPinMigrationThread domainPinMigrationThread; virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -XXX,XX +XXX,XX @@ virDomainMigrateGetMigrationPids(virDomainPtr domain, virDispatchError(domain->conn); return NULL; } + +/** + * virDomainPinMigrationThread: + * @domain: a domain object + * @cpumap: pointer to a bit map of real CPUs (in 8-bit bytes) (IN) + * Each bit set to 1 means that corresponding CPU is usable. + * Bytes are stored in little-endian order: CPU0-7, 8-15... + * In each byte, lowest CPU number is least significant bit. + * @maplen: number of bytes in cpumap, from 1 up to size of CPU map in + * underlying virtualization system (Xen...). + * If maplen < size, missing bytes are set to zero. + * If maplen > size, failure code is returned. + * + * Allocate the real CPUs to the migrationThread which will be launched + * at the beginning of migration. + * This allocation can be handled whether before the migration, or during + * the migration which performs as an instant change. + * + * This function may require privileged access to the hypervisor. + * + * Returns 0 in case of success, -1 in case of failure. + * + * Since: 9.1.0 + */ +int +virDomainPinMigrationThread(virDomainPtr domain, + unsigned char *cpumap, + int maplen) +{ + virConnectPtr conn; + g_autofree char *str = NULL; + + VIR_DOMAIN_DEBUG(domain, "migration: cpumap=%p, maplen=%d", + cpumap, maplen); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckNonNullArgGoto(cpumap, error); + virCheckPositiveArgGoto(maplen, error); + + str = virBitmapDataFormat(cpumap, maplen); + VIR_INFO("Begin to PinMigrationThread(domain=%s): new CPU Affinity:%s", + NULLSTR(domain->name), NULLSTR(str)); + + if (conn->driver->domainPinMigrationThread) { + int ret; + ret = conn->driver->domainPinMigrationThread(domain, cpumap, maplen); + if (ret < 0) { + VIR_ERROR(_("Failed to PinMigrationThread(domain=%s), ret=%d"), + NULLSTR(domain->name), ret); + goto error; + } + VIR_INFO("Success to PinMigrationThread(domain=%s)", NULLSTR(domain->name)); + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virDomainMemoryModelTypeToString; virDomainMemoryRemove; virDomainMemorySourceTypeFromString; virDomainMemorySourceTypeToString; +virDomainMigrationIDDefFree; virDomainMouseModeTypeFromString; virDomainMouseModeTypeToString; virDomainNetAllocateActualDevice; diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -XXX,XX +XXX,XX @@ LIBVIRT_8.5.0 { LIBVIRT_9.1.0 { global: + virDomainPinMigrationThread; virDomainMigrateGetMigrationPids; } LIBVIRT_8.5.0; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -XXX,XX +XXX,XX @@ qemuDomainMigrateGetMigrationPids(virDomainPtr dom, return ret; } +static int +qemuDomainPinMigrationThread(virDomainPtr dom, + unsigned char *cpumap, + int maplen) +{ + int ret = -1; + virQEMUDriver *driver = dom->conn->privateData; + virQEMUDriverConfig *cfg = NULL; + virCaps *caps = NULL; + virDomainObj *vm = NULL; + virBitmap *pcpumap = NULL; + virCgroup *cgroup_migthread = NULL; + qemuDomainObjPrivate *priv = NULL; + int migration_id = 0; + virDomainMigrationIDDef *migration = NULL; + + cfg = virQEMUDriverGetConfig(driver); + + if (!(caps = virQEMUDriverGetCapabilities(driver, false))) + goto cleanup; + + if (!(vm = qemuDomainObjFromDomain(dom))) + goto cleanup; + + priv = vm->privateData; + + if (virDomainPinMigrationThreadEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0) + goto cleanup; + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto release; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty migration thread cpumap list for pinning")); + goto release; + } + + virBitmapFree(priv->pcpumap); + priv->pcpumap = pcpumap; + + migration = g_new0(virDomainMigrationIDDef, 1); + + if (priv->migrationPids != NULL) { + if (virStrToLong_i(priv->migrationPids, NULL, 10, &(migration->thread_id)) != 0) { + VIR_ERROR(_("migrationPid trans failure, migrationPids = %s"), priv->migrationPids); + goto endjob; + } + + if (pcpumap) { + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, + migration_id, false, &cgroup_migthread) < 0) + goto endjob; + if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for migration thread %d"), migration_id); + goto endjob; + } + } + + if (virProcessSetAffinity(migration->thread_id, pcpumap, false) < 0) + goto endjob; + } + } + + ret = 0; + goto endjob; + + release: + virBitmapFree(pcpumap); + + endjob: + virDomainObjEndJob(vm); + + cleanup: + if (cgroup_migthread) + virCgroupFree(cgroup_migthread); + virDomainMigrationIDDefFree(migration); + virDomainObjEndAPI(&vm); + virObjectUnref(caps); + virObjectUnref(cfg); + return ret; +} + static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -XXX,XX +XXX,XX @@ static virHypervisorDriver qemuHypervisorDriver = { .domainGetMessages = qemuDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -XXX,XX +XXX,XX @@ static virHypervisorDriver hypervisor_driver = { .domainGetMessages = remoteDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index XXXXXXX..XXXXXXX 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -XXX,XX +XXX,XX @@ struct remote_domain_pin_iothread_args { unsigned int flags; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; + struct remote_domain_add_iothread_args { remote_nonnull_domain dom; unsigned int iothread_id; @@ -XXX,XX +XXX,XX @@ enum remote_procedure { * @generate: both * @acl: domain:migrate */ - REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443 + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, + + /** + * @generate: both + * @acl: domain:write + */ + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index XXXXXXX..XXXXXXX 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -XXX,XX +XXX,XX @@ struct remote_domain_event_memory_device_size_change_msg { remote_nonnull_string alias; uint64_t size; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; struct remote_domain_migrate_get_migration_pids_args { remote_nonnull_domain dom; u_int flags; @@ -XXX,XX +XXX,XX @@ enum remote_procedure { REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441, REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, }; diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -XXX,XX +XXX,XX @@ virCgroupNewThread(virCgroup *domain, case VIR_CGROUP_THREAD_IOTHREAD: name = g_strdup_printf("iothread%d", id); break; + case VIR_CGROUP_THREAD_MIGRATION_THREAD: + name = g_strdup_printf("migthread%d", id); + break; case VIR_CGROUP_THREAD_LAST: virReportError(VIR_ERR_INTERNAL_ERROR, _("unexpected name value %d"), nameval); diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/vircgroup.h +++ b/src/util/vircgroup.h @@ -XXX,XX +XXX,XX @@ typedef enum { VIR_CGROUP_THREAD_VCPU = 0, VIR_CGROUP_THREAD_EMULATOR, VIR_CGROUP_THREAD_IOTHREAD, + VIR_CGROUP_THREAD_MIGRATION_THREAD, VIR_CGROUP_THREAD_LAST } virCgroupThreadName; -- 2.33.0 Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- tools/virsh-domain.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index XXXXXXX..XXXXXXX 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -XXX,XX +XXX,XX @@ cmdDomDirtyRateCalc(vshControl *ctl, const vshCmd *cmd) } +/* + * "migrationpin" command + */ +static const vshCmdInfo info_migrationpin[] = { + {.name = "help", + .data = N_("control domain migrationThreads affinity") + }, + {.name = "desc", + .data = N_("Pin domain migrationThreads to host physical CPUs.") + }, + {.name = NULL} +}; + +static const vshCmdOptDef opts_migrationpin[] = { + VIRSH_COMMON_OPT_DOMAIN_FULL(0), + {.name = "cpulist", + .type = VSH_OT_DATA, + .flags = VSH_OFLAG_REQ, + .help = N_("host cpu number(s) to set") + }, + {.name = NULL} +}; + +static bool +cmdMigrationPin(vshControl *ctl, const vshCmd *cmd) +{ + g_autoptr(virshDomain) dom = NULL; + const char *name = NULL; + const char *cpulist = NULL; + int maxcpu; + g_autofree unsigned char *cpumap = NULL; + int cpumaplen; + virshControl *priv = ctl->privData; + + dom = virshCommandOptDomain(ctl, cmd, NULL); + if (!dom) + return false; + + if (vshCommandOptStringReq(ctl, cmd, "domain", &name) < 0) + return false; + + if (vshCommandOptStringReq(ctl, cmd, "cpulist", &cpulist) < 0) + return false; + + maxcpu = virshNodeGetCPUCount(priv->conn); + if (maxcpu < 0) + return false; + + cpumap = virshParseCPUList(ctl, &cpumaplen, cpulist, maxcpu); + if (!cpumap) + return false; + + if (virDomainPinMigrationThread(dom, cpumap, + cpumaplen) != 0) + return false; + + vshPrintExtra(ctl, _("Pin domain %s's migration thread " + "to cpulist %s success.\n"), name, cpulist); + + return true; +} + + const vshCmdDef domManagementCmds[] = { {.name = "attach-device", .handler = cmdAttachDevice, @@ -XXX,XX +XXX,XX @@ const vshCmdDef domManagementCmds[] = { .info = info_domdirtyrate_calc, .flags = 0 }, + {.name = "migrationpin", + .handler = cmdMigrationPin, + .opts = opts_migrationpin, + .info = info_migrationpin, + .flags = 0 + }, {.name = NULL} }; -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> add qemu monitor callback functions Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_monitor.c | 10 ++++++++++ src/qemu/qemu_monitor.h | 7 +++++++ src/qemu/qemu_monitor_json.c | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -XXX,XX +XXX,XX @@ qemuMonitorEmitPRManagerStatusChanged(qemuMonitor *mon, } +void +qemuMonitorEmitMigrationPid(qemuMonitor *mon, + int mpid) +{ + VIR_DEBUG("mon=%p, pass=%d", mon, mpid); + + QEMU_MONITOR_CALLBACK(mon, domainMigrationPid, mon->vm, mpid); +} + + void qemuMonitorEmitRdmaGidStatusChanged(qemuMonitor *mon, const char *netdev, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -XXX,XX +XXX,XX @@ typedef void (*qemuMonitorDomainMemoryDeviceSizeChange)(qemuMonitor *mon, const char *alias, unsigned long long size); +typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitor *mon, + virDomainObj *vm, + int mcpid); + typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks; struct _qemuMonitorCallbacks { qemuMonitorEofNotifyCallback eofNotify; @@ -XXX,XX +XXX,XX @@ struct _qemuMonitorCallbacks { qemuMonitorDomainMemoryFailureCallback domainMemoryFailure; qemuMonitorDomainMemoryDeviceSizeChange domainMemoryDeviceSizeChange; qemuMonitorDomainDeviceUnplugErrCallback domainDeviceUnplugError; + qemuMonitorDomainMigrationPidCallback domainMigrationPid; }; qemuMonitor *qemuMonitorOpen(virDomainObj *vm, @@ -XXX,XX +XXX,XX @@ void qemuMonitorEmitMigrationStatus(qemuMonitor *mon, void qemuMonitorEmitMigrationPass(qemuMonitor *mon, int pass); +void qemuMonitorEmitMigrationPid(qemuMonitor *mon, int mpid); + void qemuMonitorEmitAcpiOstInfo(qemuMonitor *mon, const char *alias, const char *slotType, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -XXX,XX +XXX,XX @@ static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitor *mon, virJSONV static void qemuMonitorJSONHandleMemoryFailure(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleMemoryDeviceSizeChange(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleDeviceUnplugErr(qemuMonitor *mon, virJSONValue *data); +static void qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, virJSONValue *data); typedef struct { const char *type; @@ -XXX,XX +XXX,XX @@ static qemuEventHandler eventHandlers[] = { { "MEMORY_FAILURE", qemuMonitorJSONHandleMemoryFailure, }, { "MIGRATION", qemuMonitorJSONHandleMigrationStatus, }, { "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, }, + { "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, }, { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, }, { "PR_MANAGER_STATUS_CHANGED", qemuMonitorJSONHandlePRManagerStatusChanged, }, { "RDMA_GID_STATUS_CHANGED", qemuMonitorJSONHandleRdmaGidStatusChanged, }, @@ -XXX,XX +XXX,XX @@ static qemuEventHandler eventHandlers[] = { /* We use bsearch, so keep this list sorted. */ }; +static void +qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, + virJSONValue *data) +{ + int mpid; + + if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) { + VIR_WARN("missing migration pid in migration-pid event"); + return; + } + + qemuMonitorEmitMigrationPid(mon, mpid); +} + static int qemuMonitorEventCompare(const void *key, const void *elt) { -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> Add a migrationpin to the migration parameters of live migration to bind cores to the migration thread during VM migration. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 10 ++++++++++ src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_migration.c | 3 +++ src/qemu/qemu_migration.h | 1 + src/qemu/qemu_migration_params.c | 21 +++++++++++++++++++++ src/qemu/qemu_migration_params.h | 4 ++++ 7 files changed, 41 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index XXXXXXX..XXXXXXX 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -XXX,XX +XXX,XX @@ typedef enum { */ # define VIR_MIGRATE_PARAM_TLS_DESTINATION "tls.destination" +/** + * VIR_MIGRATE_PARAM_MIGRATIONPIN: + * + * virDomainMigrate* params field: the pin of migration threads for + * migration as VIR_TYPED_PARAM_STRING. + * + * Since: 9.1.0 + */ +# define VIR_MIGRATE_PARAM_MIGRATIONPIN "migration.pin" + /* Domain migration. */ virDomainPtr virDomainMigrate (virDomainPtr domain, virConnectPtr dconn, unsigned long flags, const char *dname, diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -XXX,XX +XXX,XX @@ qemuDomainObjPrivateFree(void *data) g_free(priv->lockState); g_free(priv->origname); g_free(priv->migrationPids); + g_free(priv->migrationThreadPinList); virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -XXX,XX +XXX,XX @@ struct _qemuDomainObjPrivate { int nbdPort; /* Port used for migration with NBD */ unsigned short migrationPort; char *migrationPids; + char *migrationThreadPinList; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case it was changed for the current diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -XXX,XX +XXX,XX @@ qemuMigrationDstPrepareFresh(virQEMUDriver *driver, priv = vm->privateData; priv->origname = g_strdup(origname); + g_free(priv->migrationPids); if (taint_hook) { /* Domain XML has been altered by a hook script. */ @@ -XXX,XX +XXX,XX @@ qemuMigrationSrcRun(virQEMUDriver *driver, priv->migMaxBandwidth * 1024 * 1024) < 0) goto error; + qemuMigrationMigrationParamsToVM(migParams, vm); + if (qemuMigrationParamsApply(vm, VIR_ASYNC_JOB_MIGRATION_OUT, migParams, flags) < 0) goto error; diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -XXX,XX +XXX,XX @@ VIR_MIGRATE_PARAM_COMPRESSION_MT_LEVEL, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_THREADS, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_DTHREADS, VIR_TYPED_PARAM_INT, \ + VIR_MIGRATE_PARAM_MIGRATIONPIN, VIR_TYPED_PARAM_STRING, \ VIR_MIGRATE_PARAM_COMPRESSION_XBZRLE_CACHE, VIR_TYPED_PARAM_ULLONG, \ VIR_MIGRATE_PARAM_PERSIST_XML, VIR_TYPED_PARAM_STRING, \ VIR_MIGRATE_PARAM_AUTO_CONVERGE_INITIAL, VIR_TYPED_PARAM_INT, \ diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -XXX,XX +XXX,XX @@ VIR_ENUM_IMPL(qemuMigrationParam, "compress-level", "compress-threads", "decompress-threads", + "migrationpin", "cpu-throttle-initial", "cpu-throttle-increment", "tls-creds", @@ -XXX,XX +XXX,XX @@ static const qemuMigrationParamsTPMapItem qemuMigrationParamsTPMap[] = { .param = QEMU_MIGRATION_PARAM_DECOMPRESS_THREADS, .party = QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION}, + {.typedParam = VIR_MIGRATE_PARAM_MIGRATIONPIN, + .param = QEMU_MIGRATION_PARAM_MIGRATIONPIN, + .party = QEMU_MIGRATION_SOURCE}, + {.typedParam = VIR_MIGRATE_PARAM_COMPRESSION_XBZRLE_CACHE, .param = QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE, .party = QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION}, @@ -XXX,XX +XXX,XX @@ static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = { [QEMU_MIGRATION_PARAM_DECOMPRESS_THREADS] = { .type = QEMU_MIGRATION_PARAM_TYPE_INT, }, + [QEMU_MIGRATION_PARAM_MIGRATIONPIN] = { + .type = QEMU_MIGRATION_PARAM_TYPE_STRING, + }, [QEMU_MIGRATION_PARAM_THROTTLE_INITIAL] = { .type = QEMU_MIGRATION_PARAM_TYPE_INT, }, @@ -XXX,XX +XXX,XX @@ qemuMigrationParamsSetCompression(virTypedParameterPtr params, return 0; } +void +qemuMigrationMigrationParamsToVM(const qemuMigrationParams *migParams, const virDomainObj *vm) +{ + if (migParams && migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].set) { + qemuDomainObjPrivate *priv = vm->privateData; + priv->migrationThreadPinList = g_strdup(migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].value.s); + } +} + void qemuMigrationParamsSetBlockDirtyBitmapMapping(qemuMigrationParams *migParams, @@ -XXX,XX +XXX,XX @@ qemuMigrationParamsToJSON(qemuMigrationParams *migParams, if (!pv->set) continue; + if (i == QEMU_MIGRATION_PARAM_MIGRATIONPIN) { + continue; + } + if (postcopyResume && !qemuMigrationParamInfo[i].applyOnPostcopyResume) continue; diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -XXX,XX +XXX,XX @@ typedef enum { QEMU_MIGRATION_PARAM_COMPRESS_LEVEL, QEMU_MIGRATION_PARAM_COMPRESS_THREADS, QEMU_MIGRATION_PARAM_DECOMPRESS_THREADS, + QEMU_MIGRATION_PARAM_MIGRATIONPIN, QEMU_MIGRATION_PARAM_THROTTLE_INITIAL, QEMU_MIGRATION_PARAM_THROTTLE_INCREMENT, QEMU_MIGRATION_PARAM_TLS_CREDS, @@ -XXX,XX +XXX,XX @@ typedef enum { virBitmap * qemuMigrationParamsGetAlwaysOnCaps(qemuMigrationParty party); +void +qemuMigrationMigrationParamsToVM(const qemuMigrationParams *migParams, const virDomainObj *vm); + qemuMigrationParams * qemuMigrationParamsFromFlags(virTypedParameterPtr params, int nparams, -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> Add qemuProcessGetPcpumap to get cpumap from migration parameters when 'virsh migrationpin' is not called. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_process.c | 79 +++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 4 +++ 2 files changed, 83 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ qemuProcessRecoverMigration(virQEMUDriver *driver, } +unsigned char * +virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu) +{ + int lastcpu; + unsigned char *cpumap = NULL; + virBitmap *map = NULL; + + if (cpulist[0] == 'r') { + map = virBitmapNew(maxcpu); + if (!map) + return NULL; + virBitmapSetAll(map); + } else { + if (virBitmapParse(cpulist, &map, 1024) < 0 || + virBitmapIsAllClear(map)) { + goto cleanup; + } + + lastcpu = virBitmapLastSetBit(map); + if (lastcpu >= maxcpu) + goto cleanup; + } + + if (virBitmapToData(map, &cpumap, cpumaplen) < 0) + VIR_ERROR(_("Bitmap to data failure")); + + cleanup: + virBitmapFree(map); + return cpumap; +} + + +/* + * The value of "virsh migrationpin" is saved to priv->pcpumap + * If priv->pcpumap is NULL, it means migrationpin command is not called, + * otherwise we set the affinity of migration thread by migrationpin + */ +static virBitmap* +qemuProcessGetPcpumap(qemuDomainObjPrivate *priv) +{ + int cpumaplen = 0; + int maxcpu = 0; + virBitmap *pcpumap = NULL; + g_autofree unsigned char *cpumap = NULL; + + if (priv->pcpumap) + return priv->pcpumap; + + if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) { + VIR_ERROR(_("didn't set the migratin thread pin")); + return NULL; + } + + /* judge whether set_migration_pin is default value or not */ + if (STREQ(priv->migrationThreadPinList, "none")) + return NULL; + + maxcpu = virHostCPUGetCount(); + if (maxcpu < 0) { + VIR_ERROR(_("get the cpu count of host failure")); + return NULL; + } + + cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu); + if (!cpumap) { + VIR_ERROR(_("parse migration.pin param failure : migration.pin = %s"), + priv->migrationThreadPinList); + return NULL; + } + + if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) { + VIR_ERROR(_("Bitmap data failure")); + return NULL; + } + + return pcpumap; +} + + static int qemuProcessRecoverJob(virQEMUDriver *driver, virDomainObj *vm, diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -XXX,XX +XXX,XX @@ bool qemuProcessRebootAllowed(const virDomainDef *def); void qemuProcessCleanupMigrationJob(virQEMUDriver *driver, virDomainObj *vm); + +unsigned char *virParseCPUList(int *cpumaplen, + const char *cpulist, + int maxcpu); -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> add domainMigrationPid for qemuMonitorCallbacks Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_process.c | 107 ++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 7 +++ 2 files changed, 114 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ static qemuMonitorCallbacks monitorCallbacks = { .domainMemoryFailure = qemuProcessHandleMemoryFailure, .domainMemoryDeviceSizeChange = qemuProcessHandleMemoryDeviceSizeChange, .domainDeviceUnplugError = qemuProcessHandleDeviceUnplugErr, + .domainMigrationPid = qemuProcessHandleMigrationPid, }; static void @@ -XXX,XX +XXX,XX @@ qemuProcessRecoverMigration(virQEMUDriver *driver, } +int +qemuProcessSetupMigration(virDomainObj *vm, + virDomainMigrationIDDef *migration) +{ + return qemuProcessSetupPid(vm, migration->thread_id, + VIR_CGROUP_THREAD_MIGRATION_THREAD, + 0, + vm->def->cputune.emulatorpin, + vm->def->cputune.emulator_period, + vm->def->cputune.emulator_quota, + &migration->sched); +} + + unsigned char * virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu) { @@ -XXX,XX +XXX,XX @@ qemuProcessGetPcpumap(qemuDomainObjPrivate *priv) } +/* + * In order to set migration thread affinity when vm is migrating, + * we should create the cgroup for migration thread. + */ +static void +qemuProcessSetMigthreadAffinity(qemuDomainObjPrivate *priv, + virBitmap *pcpumap, + int mpid) +{ + int migration_id = 0; + virCgroup *cgroup_migthread = NULL; + + if (!pcpumap) + return; + + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, + migration_id, false, &cgroup_migthread) < 0) + goto cleanup; + + if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for migration%d thread"), migration_id); + goto cleanup; + } + } + + if (virProcessSetAffinity(mpid, pcpumap, false) < 0) + VIR_WARN("failed to set affinity in migration"); + + cleanup: + if (cgroup_migthread) + virCgroupFree(cgroup_migthread); + return; +} + + +int +qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid) +{ + qemuDomainObjPrivate *priv = NULL; + char *mpidStr = NULL; + virDomainMigrationIDDef *migration = NULL; + virBitmap *pcpumap = NULL; + virObjectLock(vm); + + VIR_INFO("Migrating domain %p %s, migration pid %d", + vm, vm->def->name, mpid); + + priv = vm->privateData; + if (vm->job->asyncJob == VIR_ASYNC_JOB_NONE) { + VIR_DEBUG("got MIGRATION_PID event without a migration job"); + goto cleanup; + } + + migration = g_new0(virDomainMigrationIDDef, 1); + migration->thread_id = mpid; + + if (qemuProcessSetupMigration(vm, migration) < 0) { + VIR_ERROR(_("fail to setup migration cgroup")); + goto cleanup; + } + + mpidStr = g_strdup_printf("%d", mpid); + g_free(priv->migrationPids); + priv->migrationPids = mpidStr; + + pcpumap = qemuProcessGetPcpumap(priv); + + if (!pcpumap) + goto cleanup; + + qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid); + + cleanup: + /* + * If the value of pcpumap is setted by priv->migrationThreadPinList, + * we need to free pcpumap. + */ + if (pcpumap != priv->pcpumap) + virBitmapFree(pcpumap); + virDomainMigrationIDDefFree(migration); + virObjectUnlock(vm); + + return 0; +} + + static int qemuProcessRecoverJob(virQEMUDriver *driver, virDomainObj *vm, diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -XXX,XX +XXX,XX @@ void qemuProcessCleanupMigrationJob(virQEMUDriver *driver, unsigned char *virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu); + +int qemuProcessSetupMigration(virDomainObj *vm, + virDomainMigrationIDDef *migration); + +int qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid); -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> Firstly, we need to get multifd pids, add virDomainMigrateGetMigrationMultiFdPids() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 2 ++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 40 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 1 + src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 33 ++++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 15 ++++++++++++ 10 files changed, 117 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index XXXXXXX..XXXXXXX 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -XXX,XX +XXX,XX @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, char *virDomainMigrateGetMigrationPids(virDomainPtr domain, unsigned int flags); +char *virDomainMigrateGetMigrationMultiFdPids(virDomainPtr domain, + unsigned int flags); #endif /* LIBVIRT_DOMAIN_H */ diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index XXXXXXX..XXXXXXX 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -XXX,XX +XXX,XX @@ typedef char * (*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, unsigned int flags); +typedef char * +(*virDrvDomainMigrateGetMigrationMultiFdPids)(virDomainPtr domain, + unsigned int flags); + typedef struct _virHypervisorDriver virHypervisorDriver; /** @@ -XXX,XX +XXX,XX @@ struct _virHypervisorDriver { virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; virDrvDomainPinMigrationThread domainPinMigrationThread; virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; + virDrvDomainMigrateGetMigrationMultiFdPids domainMigrateGetMigrationMultiFdPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -XXX,XX +XXX,XX @@ virDomainMigrateGetMigrationPids(virDomainPtr domain, return NULL; } + +/* + * virDomainMigrateGetMigrationMultiFdPids: + * @domain: pointer to domain object + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Get multifd thread pid. + * + * Returns the multifd1 and multifd2 pids which must be freed by the caller, or + * NULL if there was an error. + * + * Since: 9.1.0 + */ +char * +virDomainMigrateGetMigrationMultiFdPids(virDomainPtr domain, + unsigned int flags) +{ + virConnectPtr conn; + VIR_DOMAIN_DEBUG(domain, "MultiFd pids flags=0x%x", flags); + virResetLastError(); + virCheckDomainReturn(domain, NULL); + conn = domain->conn; + + virCheckReadOnlyGoto(domain->conn->flags, error); + + if (conn->driver->domainMigrateGetMigrationMultiFdPids) { + char *ret; + ret = conn->driver->domainMigrateGetMigrationMultiFdPids(domain, flags); + if (!ret) + goto error; + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return NULL; +} + /** * virDomainPinMigrationThread: * @domain: a domain object diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -XXX,XX +XXX,XX @@ LIBVIRT_9.1.0 { global: virDomainPinMigrationThread; virDomainMigrateGetMigrationPids; + virDomainMigrateGetMigrationMultiFdPids; } LIBVIRT_8.5.0; # .... define new API here using predicted next version number .... diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -XXX,XX +XXX,XX @@ qemuDomainObjPrivateFree(void *data) g_free(priv->origname); g_free(priv->migrationPids); g_free(priv->migrationThreadPinList); + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdCount = 0; virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -XXX,XX +XXX,XX @@ struct _qemuDomainObjPrivate { unsigned short migrationPort; char *migrationPids; char *migrationThreadPinList; + char *migrationMultiFdPids; + unsigned int migrationMultiFdCount; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case it was changed for the current diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -XXX,XX +XXX,XX @@ qemuDomainMigrateGetMigrationPids(virDomainPtr dom, return ret; } + +static char * +qemuDomainMigrateGetMigrationMultiFdPids(virDomainPtr dom, + unsigned int flags) +{ + char *ret = NULL; + virDomainObj *vm = NULL; + qemuDomainObjPrivate *priv = NULL; + + virCheckFlags(0, NULL); + + vm = qemuDomainObjFromDomain(dom); + if (!vm) + goto cleanup; + + if (virDomainMigrateGetMigrationMultiFdPidsEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + priv = vm->privateData; + + if (priv->migrationMultiFdPids) { + ret = g_strdup(priv->migrationMultiFdPids); + if (!ret) + VIR_ERROR(_("failed to strdup MultiFdPids")); + } + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + + static int qemuDomainPinMigrationThread(virDomainPtr dom, unsigned char *cpumap, @@ -XXX,XX +XXX,XX @@ static virHypervisorDriver qemuHypervisorDriver = { .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ + .domainMigrateGetMigrationMultiFdPids = qemuDomainMigrateGetMigrationMultiFdPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -XXX,XX +XXX,XX @@ static virHypervisorDriver hypervisor_driver = { .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ + .domainMigrateGetMigrationMultiFdPids = remoteDomainMigrateGetMigrationMultiFdPids, /* 9.1.0 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index XXXXXXX..XXXXXXX 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -XXX,XX +XXX,XX @@ struct remote_domain_migrate_get_migration_pids_ret { remote_nonnull_string migrationPids; }; +struct remote_domain_migrate_get_migration_multi_fd_pids_args { + remote_nonnull_domain dom; + unsigned int flags; +}; + +struct remote_domain_migrate_get_migration_multi_fd_pids_ret { + remote_nonnull_string migrationMultiFdPids; +}; + struct remote_domain_migrate_set_max_speed_args { remote_nonnull_domain dom; unsigned hyper bandwidth; @@ -XXX,XX +XXX,XX @@ enum remote_procedure { * @generate: both * @acl: domain:write */ - REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_MULTI_FD_PIDS = 445 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index XXXXXXX..XXXXXXX 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -XXX,XX +XXX,XX @@ struct remote_domain_migrate_get_migration_pids_args { struct remote_domain_migrate_get_migration_pids_ret { remote_nonnull_string migrationPids; }; +struct remote_domain_migrate_get_migration_pids_args { + remote_nonnull_domain dom; + u_int flags; +}; +struct remote_domain_migrate_get_migration_pids_ret { + remote_nonnull_string migrationPids; +}; +struct remote_domain_migrate_get_migration_multi_fd_pids_args { + remote_nonnull_domain dom; + u_int flags; +}; +struct remote_domain_migrate_get_migration_multi_fd_pids_ret { + remote_nonnull_string m igrationMultiFdPids; +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -XXX,XX +XXX,XX @@ enum remote_procedure { REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_MULTI_FD_PIDS = 445, }; -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> Secondly, we start to pin multifd pid by given cpumap. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_driver.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -XXX,XX +XXX,XX @@ qemuDomainPinMigrationThread(virDomainPtr dom, qemuDomainObjPrivate *priv = NULL; int migration_id = 0; virDomainMigrationIDDef *migration = NULL; + char **multiFdPids = NULL; + unsigned int multiFdIndex = 0; cfg = virQEMUDriverGetConfig(driver); @@ -XXX,XX +XXX,XX @@ qemuDomainPinMigrationThread(virDomainPtr dom, } } + /* + * set migration multiFd thread affinity + */ + if (priv->migrationMultiFdPids != NULL && pcpumap) { + multiFdPids = g_strsplit(priv->migrationMultiFdPids, "/", + priv->migrationMultiFdCount); + for (multiFdIndex = 0; multiFdIndex < (priv->migrationMultiFdCount); multiFdIndex++) { + if (virStrToLong_i(multiFdPids[multiFdIndex], NULL, 10, &(migration->thread_id)) != 0) { + VIR_ERROR(_("migrationMultiFdPids trans failure, migrationMultiFdPid = %s"), + multiFdPids[multiFdIndex]); + goto endjob; + } + if (virProcessSetAffinity(migration->thread_id, pcpumap, false) < 0) + goto endjob; + } + } + ret = 0; goto endjob; @@ -XXX,XX +XXX,XX @@ qemuDomainPinMigrationThread(virDomainPtr dom, virDomainObjEndAPI(&vm); virObjectUnref(caps); virObjectUnref(cfg); + g_strfreev(multiFdPids); return ret; } -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> add qemu monitor callback functions Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_monitor.c | 10 ++++++++++ src/qemu/qemu_monitor.h | 6 ++++++ src/qemu/qemu_monitor_json.c | 16 ++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -XXX,XX +XXX,XX @@ qemuMonitorEmitMigrationPid(qemuMonitor *mon, } +void +qemuMonitorEmitMigrationMultiFdPids(qemuMonitor *mon, + int mpid) +{ + VIR_DEBUG("mon=%p, pass=%d", mon, mpid); + + QEMU_MONITOR_CALLBACK(mon, domainMigrationMultiFdPids, mon->vm, mpid); +} + + void qemuMonitorEmitRdmaGidStatusChanged(qemuMonitor *mon, const char *netdev, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -XXX,XX +XXX,XX @@ typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitor *mon, virDomainObj *vm, int mcpid); +typedef int (*qemuMonitorDomainMigrationMultiFdPidsCallback)(qemuMonitor *mon, + virDomainObj *vm, + int mcpid); + typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks; struct _qemuMonitorCallbacks { qemuMonitorEofNotifyCallback eofNotify; @@ -XXX,XX +XXX,XX @@ struct _qemuMonitorCallbacks { qemuMonitorDomainMemoryDeviceSizeChange domainMemoryDeviceSizeChange; qemuMonitorDomainDeviceUnplugErrCallback domainDeviceUnplugError; qemuMonitorDomainMigrationPidCallback domainMigrationPid; + qemuMonitorDomainMigrationMultiFdPidsCallback domainMigrationMultiFdPids; }; qemuMonitor *qemuMonitorOpen(virDomainObj *vm, @@ -XXX,XX +XXX,XX @@ void qemuMonitorEmitMigrationPass(qemuMonitor *mon, int pass); void qemuMonitorEmitMigrationPid(qemuMonitor *mon, int mpid); +void qemuMonitorEmitMigrationMultiFdPids(qemuMonitor *mon, int mpid); void qemuMonitorEmitAcpiOstInfo(qemuMonitor *mon, const char *alias, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -XXX,XX +XXX,XX @@ static void qemuMonitorJSONHandleMemoryFailure(qemuMonitor *mon, virJSONValue *d static void qemuMonitorJSONHandleMemoryDeviceSizeChange(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleDeviceUnplugErr(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, virJSONValue *data); +static void qemuMonitorJSONHandleMigrationMultiFdPids(qemuMonitor *mon, virJSONValue *data); typedef struct { const char *type; @@ -XXX,XX +XXX,XX @@ static qemuEventHandler eventHandlers[] = { { "MEMORY_DEVICE_SIZE_CHANGE", qemuMonitorJSONHandleMemoryDeviceSizeChange, }, { "MEMORY_FAILURE", qemuMonitorJSONHandleMemoryFailure, }, { "MIGRATION", qemuMonitorJSONHandleMigrationStatus, }, + { "MIGRATION_MULTIFD_PID", qemuMonitorJSONHandleMigrationMultiFdPids, }, { "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, }, { "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, }, { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, }, @@ -XXX,XX +XXX,XX @@ qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, qemuMonitorEmitMigrationPid(mon, mpid); } +static void +qemuMonitorJSONHandleMigrationMultiFdPids(qemuMonitor *mon, + virJSONValue *data) +{ + int mpid; + + if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) { + VIR_WARN("missing multifd pid in migration-multifd-pid event"); + return; + } + + qemuMonitorEmitMigrationMultiFdPids(mon, mpid); +} + static int qemuMonitorEventCompare(const void *key, const void *elt) { -- 2.33.0 From: zhengchuan <zhengchuan@huawei.com> support migration multifd thread pin by configuration. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_migration.c | 2 ++ src/qemu/qemu_process.c | 61 +++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 4 +++ 3 files changed, 67 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -XXX,XX +XXX,XX @@ qemuMigrationDstPrepareFresh(virQEMUDriver *driver, priv = vm->privateData; priv->origname = g_strdup(origname); g_free(priv->migrationPids); + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdCount = 0; if (taint_hook) { /* Domain XML has been altered by a hook script. */ diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ static qemuMonitorCallbacks monitorCallbacks = { .domainMemoryDeviceSizeChange = qemuProcessHandleMemoryDeviceSizeChange, .domainDeviceUnplugError = qemuProcessHandleDeviceUnplugErr, .domainMigrationPid = qemuProcessHandleMigrationPid, + .domainMigrationMultiFdPids = qemuProcessHandleMigrationMultiFdPids, }; static void @@ -XXX,XX +XXX,XX @@ qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, return 0; } +int +qemuProcessHandleMigrationMultiFdPids(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid) +{ + qemuDomainObjPrivate *priv = NULL; + char *mpidOldStr = NULL; + char *mpidStr = NULL; + virDomainMigrationIDDef *migration = NULL; + virBitmap *pcpumap = NULL; + virObjectLock(vm); + + VIR_INFO("Migrating domain %p %s, migration-multifd pid %d", + vm, vm->def->name, mpid); + + priv = vm->privateData; + if (vm->job->asyncJob == VIR_ASYNC_JOB_NONE) { + VIR_DEBUG("got MIGRATION_MULTIFD_PID event without a migration job"); + goto cleanup; + } + + migration = g_new0(virDomainMigrationIDDef, 1); + migration->thread_id = mpid; + + if (qemuProcessSetupMigration(vm, migration) < 0) { + VIR_ERROR(_("fail to setup migration multiFd cgroup")); + goto cleanup; + } + + mpidOldStr = priv->migrationMultiFdPids; + if (!mpidOldStr) { + mpidStr = g_strdup_printf("%d", mpid); + } else { + mpidStr = g_strdup_printf("%s/%d", mpidOldStr, mpid); + } + + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdPids = mpidStr; + priv->migrationMultiFdCount++; + + pcpumap = qemuProcessGetPcpumap(priv); + + if (!pcpumap) + goto cleanup; + + qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid); + + cleanup: + /* + * If the value of pcpumap is setted by priv->migrationThreadPinList, + * we need to free pcpumap. + */ + if (pcpumap != priv->pcpumap) + virBitmapFree(pcpumap); + virDomainMigrationIDDefFree(migration); + virObjectUnlock(vm); + + return 0; +} + static int qemuProcessRecoverJob(virQEMUDriver *driver, diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -XXX,XX +XXX,XX @@ int qemuProcessSetupMigration(virDomainObj *vm, int qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, virDomainObj *vm, int mpid); + +int qemuProcessHandleMigrationMultiFdPids(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid); -- 2.33.0 Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_process.c | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ qemuProcessHandleSpiceMigrated(qemuMonitor *mon G_GNUC_UNUSED, } +static void +qemuProcessHandleMigrationPinStatus(virDomainObj *vm, int status) +{ + qemuDomainObjPrivate *priv = vm->privateData; + + if (vm->job->asyncJob != VIR_ASYNC_JOB_MIGRATION_OUT) + return; + + switch (status) { + case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_SETUP: + case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER: + case QEMU_MONITOR_MIGRATION_STATUS_DEVICE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY_PAUSED: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY_RECOVER: + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: + case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED: + case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG: + break; + case QEMU_MONITOR_MIGRATION_STATUS_ERROR: + /* + * migration thread is still running, + * so we can't delete migration Cgroup. + */ + g_free(priv->migrationPids); + g_free(priv->migrationMultiFdPids); + g_free(priv->migrationThreadPinList); + priv->migrationMultiFdCount = 0; + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + break; + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED: + g_free(priv->migrationPids); + g_free(priv->migrationMultiFdPids); + g_free(priv->migrationThreadPinList); + priv->migrationMultiFdCount = 0; + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + if (virCgroupDelThread(priv->cgroup, + VIR_CGROUP_THREAD_MIGRATION_THREAD, 0) < 0) + VIR_WARN("Failed to delete migration thread Cgroup!"); + VIR_INFO("success to free pcpumap and migrationPids"); + break; + default: + VIR_WARN("got unknown migration status'%s'", + qemuMonitorMigrationStatusTypeToString(status)); + break; + } + + return; +} + + static void qemuProcessHandleMigrationStatus(qemuMonitor *mon G_GNUC_UNUSED, virDomainObj *vm, @@ -XXX,XX +XXX,XX @@ qemuProcessHandleMigrationStatus(qemuMonitor *mon G_GNUC_UNUSED, default: break; } + qemuProcessHandleMigrationPinStatus(vm, status); cleanup: virObjectUnlock(vm); -- 2.33.0 Support set the CPU affinity of the live migration threads to improve the migration performance in specific cases. By default, the migration thread shares CPU resources with the VM process. With those API, support pin migration thread to expected CPU list to avoid preempting CPU resources of VM process. Use the following methods to implement migration thread pin: - set the cpulist by virsh command 'virsh migrationpin' or migration param 'migration.pin'. These settings affect only the next migration - continuously query migration thread information[1] during the migration process until matches the thread number desired or migration finished unexpectedly - pin migration thread to expected cpulist, we can set/change the cpulist by 'virsh migrationpin' during migration at anytime Note that only support pin main thread of migration(live migration) and parallel send threads now. diff to v1: - add qmp command to query migration thread information - change the method of get qemu's migration thread info, from qemu event to qmp commands. - unified the interface of common migration and parallel migration. ref: [1]https://patchew.org/QEMU/20230203073519.2969147-1-jiangjiacheng@huawei.com/ Jiang Jiacheng (8): migration/migration-pin: Implemente the interface of obtaining migration thread information migration/migration-pin: get migration pid for migration pin migration/migration-pin: Introduce virDomainPinMigrationThread API migration/migration-pin: Implement qemuDomainPinMigrationThread migration/migration-pin: support migration pin by virsh command migration/migration-pin: add migrationpin for migration parameters migration/migration-pin: pin migration thread immediately after get thread info migration/migration-pin/multifd-pin: add migration pin status handle include/libvirt/libvirt-domain.h | 15 +++ src/conf/domain_conf.c | 9 ++ src/conf/domain_conf.h | 10 ++ src/conf/virconftypes.h | 2 + src/driver-hypervisor.h | 6 + src/libvirt-domain.c | 65 ++++++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 4 + src/qemu/qemu_domain.c | 26 ++++ src/qemu/qemu_domain.h | 13 ++ src/qemu/qemu_driver.c | 56 +++++++++ src/qemu/qemu_migration.c | 111 +++++++++++++++++ src/qemu/qemu_migration.h | 33 +++--- src/qemu/qemu_migration_params.c | 40 +++++++ src/qemu/qemu_migration_params.h | 9 ++ src/qemu/qemu_monitor.c | 25 ++++ src/qemu/qemu_monitor.h | 11 ++ src/qemu/qemu_monitor_json.c | 62 ++++++++++ src/qemu/qemu_monitor_json.h | 5 + src/qemu/qemu_process.c | 198 +++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 13 ++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 +- src/remote_protocol-structs | 5 + src/util/vircgroup.c | 3 + src/util/vircgroup.h | 1 + tools/virsh-domain.c | 69 +++++++++++ 27 files changed, 789 insertions(+), 17 deletions(-) -- 2.33.0 First, we need get migration thread info, implement the related interface to obtain migration threadinfo from qemu using qmp command. Implementation of the Interface for Obtaining Thread Information Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_domain.c | 24 ++++++++++++++ src/qemu/qemu_domain.h | 11 +++++++ src/qemu/qemu_monitor.c | 25 +++++++++++++++ src/qemu/qemu_monitor.h | 11 +++++++ src/qemu/qemu_monitor_json.c | 62 ++++++++++++++++++++++++++++++++++++ src/qemu/qemu_monitor_json.h | 5 +++ 6 files changed, 138 insertions(+) diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -XXX,XX +XXX,XX @@ qemuDomainSecretInfoDestroy(qemuDomainSecretInfo *secinfo) qemuDomainSecretInfoClear(secinfo, true); } +static void +qemuDomainMigThreadInfoFree(qemuDomainMigThreadInfo *migthread) +{ + if (!migthread) + return; + + g_free(migthread->thread_name); +} + +void +qemuDomainMigThreadsInfoFree(qemuDomainMigThreadInfo **migthreads, + int nmigthreads) +{ + size_t i; + if (!migthreads) + return; + + for (i = 0; i < nmigthreads; i++) { + g_clear_pointer(&migthreads[i], qemuDomainMigThreadInfoFree); + } + g_free(migthreads); +} static virClass *qemuDomainDiskPrivateClass; static void qemuDomainDiskPrivateDispose(void *obj); @@ -XXX,XX +XXX,XX @@ qemuDomainObjPrivateDataClear(qemuDomainObjPrivate *priv) priv->allowReboot = VIR_TRISTATE_BOOL_ABSENT; g_clear_pointer(&priv->migrationCaps, virBitmapFree); + qemuDomainMigThreadsInfoFree(priv->migThreadsInfo, priv->migThreadCount); + priv->migThreadCount = 0; virHashRemoveAll(priv->blockjobs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -XXX,XX +XXX,XX @@ struct _qemuDomainSecretInfo { char *ciphertext; /* encoded/encrypted secret */ }; +typedef struct _qemuDomainMigThreadInfo qemuDomainMigThreadInfo; +struct _qemuDomainMigThreadInfo { + char *thread_name; + int thread_id; +}; + typedef struct _qemuDomainObjPrivate qemuDomainObjPrivate; struct _qemuDomainObjPrivate { virQEMUDriver *driver; @@ -XXX,XX +XXX,XX @@ struct _qemuDomainObjPrivate { unsigned long migMaxBandwidth; char *origname; int nbdPort; /* Port used for migration with NBD */ + int migThreadCount; + qemuDomainMigThreadInfo **migThreadsInfo; unsigned short migrationPort; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case @@ -XXX,XX +XXX,XX @@ int qemuDomainSecretChardevPrepare(virQEMUDriverConfig *cfg, ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(3) ATTRIBUTE_NONNULL(4); +void qemuDomainMigThreadsInfoFree(qemuDomainMigThreadInfo **migthreads, + int nmigthreads); + void qemuDomainCleanupStorageSourceFD(virStorageSource *src); void qemuDomainStartupCleanup(virDomainObj *vm); diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -XXX,XX +XXX,XX @@ qemuMonitorGetMigrationStats(qemuMonitor *mon, return qemuMonitorJSONGetMigrationStats(mon, stats, error); } +void qemuMonitorMigThreadsInfoFree(qemuMonitorMigThreadInfo **migthreads, + int nmigthreads) +{ + size_t i; + + if (!migthreads) + return; + + for (i = 0; i < nmigthreads; i++) { + g_free(migthreads[i]->thread_name); + g_free(migthreads[i]); + } + + g_free(migthreads); +} + +int +qemuMonitorGetMigThreadsInfo(qemuMonitor *mon, + qemuMonitorMigThreadInfo ***migthreads, + int *nmigthreads) +{ + QEMU_CHECK_MONITOR(mon); + + return qemuMonitorJSONGetMigThreadsInfo(mon, migthreads, nmigthreads); +} int qemuMonitorMigrateToFd(qemuMonitor *mon, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -XXX,XX +XXX,XX @@ int qemuMonitorGetMigrationCapabilities(qemuMonitor *mon, int qemuMonitorSetMigrationCapabilities(qemuMonitor *mon, virJSONValue **caps); +typedef struct _qemuMonitorMigThreadInfo qemuMonitorMigThreadInfo; +struct _qemuMonitorMigThreadInfo { + char *thread_name; + int thread_id; +}; +int qemuMonitorGetMigThreadsInfo(qemuMonitor *mon, + qemuMonitorMigThreadInfo ***migthreads, + int *nmigthreads); +void qemuMonitorMigThreadsInfoFree(qemuMonitorMigThreadInfo **migthreads, + int nmigthreads); + int qemuMonitorGetGICCapabilities(qemuMonitor *mon, virGICCapability **capabilities); diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -XXX,XX +XXX,XX @@ int qemuMonitorJSONGetMigrationStats(qemuMonitor *mon, return 0; } +int +qemuMonitorJSONGetMigThreadsInfo(qemuMonitor *mon, + qemuMonitorMigThreadInfo ***migthreads, + int *nmigthreads) +{ + int ret = -1; + g_autoptr(virJSONValue) cmd = NULL; + g_autoptr(virJSONValue) reply = NULL; + virJSONValue *data = NULL; + qemuMonitorMigThreadInfo **infolist = NULL; + size_t n = 0; + size_t i; + + *migthreads = NULL; + + if (!(cmd = qemuMonitorJSONMakeCommand("query-migrationthreads", NULL))) + return ret; + + if (qemuMonitorJSONCommand(mon, cmd, &reply) < 0) + goto cleanup; + + if (qemuMonitorJSONCheckReply(cmd, reply, VIR_JSON_TYPE_ARRAY) < 0) + goto cleanup; + + data = virJSONValueObjectGetArray(reply, "return"); + n = virJSONValueArraySize(data); + + infolist = g_new0(qemuMonitorMigThreadInfo *, n + 1); + + for (i = 0; i < n; i++) { + virJSONValue *child = virJSONValueArrayGet(data, i); + const char *tmp; + qemuMonitorMigThreadInfo *info; + + info = g_new0(qemuMonitorMigThreadInfo, 1); + + infolist[i] = info; + + if (!(tmp = virJSONValueObjectGetString(child, "name"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("query-migrationthreads reply data was missing 'name'")); + goto cleanup; + } + info->thread_name = g_strdup(tmp); + + if (virJSONValueObjectGetNumberInt(child, "thread-id", + &info->thread_id) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("query-migrationthreads reply has malformed " + "'thread-id' data")); + goto cleanup; + } + } + + *nmigthreads = n; + *migthreads = g_steal_pointer(&infolist); + ret = 0; + + cleanup: + qemuMonitorMigThreadsInfoFree(infolist, n); + return ret; +} int qemuMonitorJSONMigrate(qemuMonitor *mon, unsigned int flags, diff --git a/src/qemu/qemu_monitor_json.h b/src/qemu/qemu_monitor_json.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_monitor_json.h +++ b/src/qemu/qemu_monitor_json.h @@ -XXX,XX +XXX,XX @@ int qemuMonitorJSONSetMigrationCapabilities(qemuMonitor *mon, virJSONValue **caps); +int +qemuMonitorJSONGetMigThreadsInfo(qemuMonitor *mon, + qemuMonitorMigThreadInfo ***migthreads, + int *nmigthreads); + int qemuMonitorJSONGetGICCapabilities(qemuMonitor *mon, virGICCapability **capabilities); -- 2.33.0 Second, query migration thread information during migration process proactively. Migration should start after all thread be created, so querying the migration thread infomation before wating for completion and terminate the query in any abnormal situation. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_migration.c | 81 ++++++++++++++++++++++++++++++++ src/qemu/qemu_migration_params.c | 19 ++++++++ src/qemu/qemu_migration_params.h | 4 ++ 3 files changed, 104 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -XXX,XX +XXX,XX @@ qemuMigrationAnyCompleted(virDomainObj *vm, return -1; } +/* Returns 0 on get migration thread info success, + * -1 on unexpected failure + */ +static int +qemuMigrationSrcWaitForThreadInfo(virDomainObj *vm, + qemuMigrationParams *migParams, + unsigned long migflags, + virDomainAsyncJob asyncJob, + virConnectPtr dconn, + unsigned int flags) +{ + int ret = -1; + qemuDomainObjPrivate *priv = vm->privateData; + qemuMonitorMigThreadInfo **infolist = NULL; + qemuDomainMigThreadInfo **migthreads = NULL; + int nmigthreads = 0; + int rc; + size_t i; + + /* migrationThreadCount = live_migration + multifd channels */ + priv->migThreadCount = qemuMigrationParamsGetMigThreadCount(migParams, + migflags); + while ((rc = qemuMigrationAnyCompleted(vm, asyncJob, + dconn, flags)) != 1) { + if (rc < 0) + return rc; + + if (qemuDomainObjEnterMonitorAsync(vm, asyncJob) < 0) + return ret; + rc = qemuMonitorGetMigThreadsInfo(priv->mon, &infolist, &nmigthreads); + qemuDomainObjExitMonitor(vm); + if (rc < 0) + goto cleanup; + + if (nmigthreads == priv->migThreadCount) { + break; + } else if (nmigthreads > priv->migThreadCount) { + VIR_WARN("migration threads is more than expected"); + goto cleanup; + } else { + struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000 * 1000ull }; + virObjectUnlock(vm); + nanosleep(&ts, NULL); + virObjectLock(vm); + } + } + + migthreads = g_new0(qemuDomainMigThreadInfo *, nmigthreads + 1); + + for (i = 0; i < nmigthreads; i++) { + qemuDomainMigThreadInfo *migthread; + migthread = g_new0(qemuDomainMigThreadInfo, nmigthreads + 1); + + migthread->thread_name = g_strdup(infolist[i]->thread_name); + migthread->thread_id = infolist[i]->thread_id; + + migthreads[i] = migthread; + } + + priv->migThreadsInfo = migthreads; + ret = 0; + + cleanup: + if (migthreads && (ret < 0)) { + qemuDomainMigThreadsInfoFree(migthreads, priv->migThreadCount); + priv->migThreadCount = 0; + } + + if (infolist) { + qemuMonitorMigThreadsInfoFree(infolist, nmigthreads); + } + + return ret; +} + /* Returns 0 on success, -2 when migration needs to be cancelled, or -1 when * QEMU reports failed migration. @@ -XXX,XX +XXX,XX @@ qemuMigrationSrcRun(virQEMUDriver *driver, if (flags & VIR_MIGRATE_POSTCOPY) waitFlags |= QEMU_MIGRATION_COMPLETED_POSTCOPY; + rc = qemuMigrationSrcWaitForThreadInfo(vm, migParams, flags, + VIR_ASYNC_JOB_MIGRATION_OUT, + dconn, waitFlags); + if (rc == -1) + goto error; + rc = qemuMigrationSrcWaitForCompletion(vm, VIR_ASYNC_JOB_MIGRATION_OUT, dconn, waitFlags); if (rc == -2) diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -XXX,XX +XXX,XX @@ qemuMigrationParamsGetTLSHostname(qemuMigrationParams *migParams) return migParams->params[QEMU_MIGRATION_PARAM_TLS_HOSTNAME].value.s; } + + +int +qemuMigrationParamsGetMigThreadCount(qemuMigrationParams *migParams, + unsigned long migflags) +{ + int nmigthreads = 1; /* live_migtion */ + + if (!(migflags & VIR_MIGRATE_PARALLEL)) { + return nmigthreads; + } else { + if (migParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set) + nmigthreads += migParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i; + else + nmigthreads += 2; /* default multifd channel is 2 */ + } + + return nmigthreads; +} diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -XXX,XX +XXX,XX @@ qemuMigrationCapsGet(virDomainObj *vm, const char * qemuMigrationParamsGetTLSHostname(qemuMigrationParams *migParams); + +int +qemuMigrationParamsGetMigThreadCount(qemuMigrationParams *migParams, + unsigned long migflags); -- 2.33.0 Then, pin migration thread with given cpumap. Introduce virDomainPinMigrationThread API, and it will be implement in next patch. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- include/libvirt/libvirt-domain.h | 5 +++ src/driver-hypervisor.h | 6 +++ src/libvirt-domain.c | 65 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 4 ++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 ++++++- src/remote_protocol-structs | 5 +++ 7 files changed, 98 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index XXXXXXX..XXXXXXX 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -XXX,XX +XXX,XX @@ int virDomainGetEmulatorPinInfo (virDomainPtr domain, int maplen, unsigned int flags); +int virDomainPinMigrationThread (virDomainPtr domain, + unsigned char *cpumap, + int maplen); + + /** * virDomainIOThreadInfo: * diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index XXXXXXX..XXXXXXX 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -XXX,XX +XXX,XX @@ typedef int int seconds, unsigned int flags); +typedef int +(*virDrvDomainPinMigrationThread)(virDomainPtr domain, + unsigned char *cpumap, + int maplen); + typedef int (*virDrvDomainFDAssociate)(virDomainPtr domain, const char *name, @@ -XXX,XX +XXX,XX @@ struct _virHypervisorDriver { virDrvDomainGetMessages domainGetMessages; virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; virDrvDomainFDAssociate domainFDAssociate; + virDrvDomainPinMigrationThread domainPinMigrationThread; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -XXX,XX +XXX,XX @@ virDomainFDAssociate(virDomainPtr domain, virDispatchError(conn); return -1; } + +/** + * virDomainPinMigrationThread: + * @domain: a domain object + * @cpumap: pointer to a bit map of real CPUs (in 8-bit bytes) (IN) + * Each bit set to 1 means that corresponding CPU is usable. + * Bytes are stored in little-endian order: CPU0-7, 8-15... + * In each byte, lowest CPU number is least significant bit. + * @maplen: number of bytes in cpumap, from 1 up to size of CPU map in + * underlying virtualization system (Xen...). + * If maplen < size, missing bytes are set to zero. + * If maplen > size, failure code is returned. + * + * Allocate the real CPUs to the migrationThread which will be launched + * at the beginning of migration. + * This allocation can be handled whether before the migration, or during + * the migration which performs as an instant change. + * + * This function may require privileged access to the hypervisor. + * + * Returns 0 in case of success, -1 in case of failure. + * + * Since: 9.1.0 + */ +int +virDomainPinMigrationThread(virDomainPtr domain, + unsigned char *cpumap, + int maplen) +{ + virConnectPtr conn; + g_autofree char *str = NULL; + + VIR_DOMAIN_DEBUG(domain, "migration: cpumap=%p, maplen=%d", + cpumap, maplen); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckNonNullArgGoto(cpumap, error); + virCheckPositiveArgGoto(maplen, error); + + str = virBitmapDataFormat(cpumap, maplen); + VIR_INFO("Begin to PinMigrationThread(domain=%s): new CPU Affinity:%s", + NULLSTR(domain->name), NULLSTR(str)); + + if (conn->driver->domainPinMigrationThread) { + int ret; + ret = conn->driver->domainPinMigrationThread(domain, cpumap, maplen); + if (ret < 0) { + VIR_ERROR(_("Failed to PinMigrationThread(domain=%s), ret=%d"), + NULLSTR(domain->name), ret); + goto error; + } + VIR_INFO("Success to PinMigrationThread(domain=%s)", NULLSTR(domain->name)); + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -XXX,XX +XXX,XX @@ LIBVIRT_9.0.0 { virDomainFDAssociate; } LIBVIRT_8.5.0; +LIBVIRT_9.1.0 { + global: + virDomainPinMigrationThread; +} LIBVIRT_9.0.0; # .... define new API here using predicted next version number .... diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -XXX,XX +XXX,XX @@ static virHypervisorDriver hypervisor_driver = { .domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ .domainFDAssociate = remoteDomainFDAssociate, /* 9.0.0 */ + .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index XXXXXXX..XXXXXXX 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -XXX,XX +XXX,XX @@ struct remote_domain_fd_associate_args { remote_nonnull_string name; unsigned int flags; }; + +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; /*----- Protocol. -----*/ /* Define the program number, protocol version and procedure numbers here. */ @@ -XXX,XX +XXX,XX @@ enum remote_procedure { * @generate: none * @acl: domain:write */ - REMOTE_PROC_DOMAIN_FD_ASSOCIATE = 443 + REMOTE_PROC_DOMAIN_FD_ASSOCIATE = 443, + + /** + * @generate: both + * @acl: domain:write + */ + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index XXXXXXX..XXXXXXX 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -XXX,XX +XXX,XX @@ struct remote_domain_fd_associate_args { remote_nonnull_string name; u_int flags; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -XXX,XX +XXX,XX @@ enum remote_procedure { REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441, REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_FD_ASSOCIATE = 443, + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, }; -- 2.33.0 Add cgroup for migation thread and implement qemuDomainPinMigrationThread to pin migraiton thread to given cpumap. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/conf/domain_conf.c | 9 ++++++ src/conf/domain_conf.h | 10 +++++++ src/conf/virconftypes.h | 2 ++ src/libvirt_private.syms | 1 + src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_driver.c | 56 ++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.c | 61 ++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 10 +++++++ src/util/vircgroup.c | 3 ++ src/util/vircgroup.h | 1 + 11 files changed, 155 insertions(+) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -XXX,XX +XXX,XX @@ virDomainWatchdogDefFind(const virDomainDef *def, return -1; } + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def) +{ + if (!def) + return; + virBitmapFree(def->cpumask); + VIR_FREE(def); +} diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index XXXXXXX..XXXXXXX 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -XXX,XX +XXX,XX @@ virDomainObjGetMessages(virDomainObj *vm, bool virDomainDefHasSpiceGraphics(const virDomainDef *def); + +struct _virDomainMigrationIDDef { + bool autofill; + int thread_id; + virBitmap *cpumask; + virDomainThreadSchedParam sched; +}; + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def); diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h index XXXXXXX..XXXXXXX 100644 --- a/src/conf/virconftypes.h +++ b/src/conf/virconftypes.h @@ -XXX,XX +XXX,XX @@ typedef struct _virDomainXMLOption virDomainXMLOption; typedef struct _virDomainXMLPrivateDataCallbacks virDomainXMLPrivateDataCallbacks; typedef struct _virDomainXenbusControllerOpts virDomainXenbusControllerOpts; + +typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef; diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index XXXXXXX..XXXXXXX 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -XXX,XX +XXX,XX @@ virDomainMemoryModelTypeToString; virDomainMemoryRemove; virDomainMemorySourceTypeFromString; virDomainMemorySourceTypeToString; +virDomainMigrationIDDefFree; virDomainMouseModeTypeFromString; virDomainMouseModeTypeToString; virDomainNetAllocateActualDevice; diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -XXX,XX +XXX,XX @@ qemuDomainObjPrivateDataClear(qemuDomainObjPrivate *priv) /* remove automatic pinning data */ g_clear_pointer(&priv->autoNodeset, virBitmapFree); g_clear_pointer(&priv->autoCpuset, virBitmapFree); + g_clear_pointer(&priv->pcpumap, virBitmapFree); g_clear_pointer(&priv->pciaddrs, virDomainPCIAddressSetFree); g_clear_pointer(&priv->usbaddrs, virDomainUSBAddressSetFree); g_clear_pointer(&priv->origCPU, virCPUDefFree); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -XXX,XX +XXX,XX @@ struct _qemuDomainObjPrivate { /* Bitmaps below hold data from the auto NUMA feature */ virBitmap *autoNodeset; virBitmap *autoCpuset; + virBitmap *pcpumap; bool signalIOError; /* true if the domain condition should be signalled on I/O error */ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -XXX,XX +XXX,XX @@ qemuDomainFDAssociate(virDomainPtr domain, return ret; } +static int +qemuDomainPinMigrationThread(virDomainPtr dom, + unsigned char *cpumap, + int maplen) +{ + int ret = -1; + virQEMUDriver *driver = dom->conn->privateData; + g_autoptr(virQEMUDriverConfig) cfg = NULL; + virDomainObj *vm = NULL; + g_autoptr(virBitmap) pcpumap = NULL; + qemuDomainObjPrivate *priv = NULL; + qemuDomainMigThreadInfo **migthreads = NULL; + size_t i; + + cfg = virQEMUDriverGetConfig(driver); + + if (!(vm = qemuDomainObjFromDomain(dom))) + goto cleanup; + + priv = vm->privateData; + + if (virDomainPinMigrationThreadEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0) + goto cleanup; + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto endjob; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty migration thread cpumap list for pinning")); + goto endjob; + } + + virBitmapFree(priv->pcpumap); + priv->pcpumap = virBitmapNewCopy(pcpumap); + migthreads = priv->migThreadsInfo; + + if (migthreads && pcpumap) { + for (i = 0; i < priv->migThreadCount; i++) { + qemuProcessSetMigThreadAffinity(priv, vm, migthreads[i]->thread_id, pcpumap); + } + } + + ret = 0; + + endjob: + virDomainObjEndJob(vm); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -XXX,XX +XXX,XX @@ static virHypervisorDriver qemuHypervisorDriver = { .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ .domainFDAssociate = qemuDomainFDAssociate, /* 9.0.0 */ + .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ }; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ qemuProcessQMPStart(qemuProcessQMP *proc) return 0; } + +int +qemuProcessSetupMigration(virDomainObj *vm, + virDomainMigrationIDDef *migration) +{ + return qemuProcessSetupPid(vm, migration->thread_id, + VIR_CGROUP_THREAD_MIGRATION_THREAD, + 0, + vm->def->cputune.emulatorpin, + vm->def->cputune.emulator_period, + vm->def->cputune.emulator_quota, + &migration->sched); +} + +/* + * In order to set migration thread affinity when vm is migrating, + * we should create the cgroup for migration thread. + */ +void +qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv, + virDomainObj *vm, + int mpid, + virBitmap *pcpumap) +{ + virDomainMigrationIDDef *migration = NULL; + int migration_id = 0; + virCgroup *cgroup_migthread = NULL; + + if (!pcpumap) + return; + + migration = g_new0(virDomainMigrationIDDef, 1); + migration->thread_id = mpid; + if (qemuProcessSetupMigration(vm, migration) < 0) { + VIR_ERROR(_("fail to setup migration cgroup")); + goto cleanup; + } + + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, + migration_id, false, &cgroup_migthread) < 0) + goto cleanup; + + if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for migration%d thread"), migration_id); + goto cleanup; + } + } + + if (virProcessSetAffinity(mpid, pcpumap, false) < 0) + VIR_WARN("failed to set affinity in migration"); + + cleanup: + if (cgroup_migthread) + virCgroupFree(cgroup_migthread); + virDomainMigrationIDDefFree(migration); + return; +} diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -XXX,XX +XXX,XX @@ bool qemuProcessRebootAllowed(const virDomainDef *def); void qemuProcessCleanupMigrationJob(virQEMUDriver *driver, virDomainObj *vm); + +int +qemuProcessSetupMigration(virDomainObj *vm, + virDomainMigrationIDDef *migration); + +void +qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv, + virDomainObj *vm, + int mpid, + virBitmap *pcpumap); diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c index XXXXXXX..XXXXXXX 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -XXX,XX +XXX,XX @@ virCgroupNewThread(virCgroup *domain, case VIR_CGROUP_THREAD_IOTHREAD: name = g_strdup_printf("iothread%d", id); break; + case VIR_CGROUP_THREAD_MIGRATION_THREAD: + name = g_strdup_printf("migthread%d", id); + break; case VIR_CGROUP_THREAD_LAST: virReportError(VIR_ERR_INTERNAL_ERROR, _("unexpected name value %d"), nameval); diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h index XXXXXXX..XXXXXXX 100644 --- a/src/util/vircgroup.h +++ b/src/util/vircgroup.h @@ -XXX,XX +XXX,XX @@ typedef enum { VIR_CGROUP_THREAD_VCPU = 0, VIR_CGROUP_THREAD_EMULATOR, VIR_CGROUP_THREAD_IOTHREAD, + VIR_CGROUP_THREAD_MIGRATION_THREAD, VIR_CGROUP_THREAD_LAST } virCgroupThreadName; -- 2.33.0 Add virsh command 'virsh migrationpin', its usage is like 'virsh migrationpin [vm] [cpulist]'. After using this command, if migration thread infomation have been got, pin it to given cpumap immediately, else wait for next migraion operation and pin migration thread after getting information. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- tools/virsh-domain.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index XXXXXXX..XXXXXXX 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -XXX,XX +XXX,XX @@ cmdDomDirtyRateCalc(vshControl *ctl, const vshCmd *cmd) } +/* + * "migrationpin" command + */ +static const vshCmdInfo info_migrationpin[] = { + {.name = "help", + .data = N_("control domain migrationThreads affinity") + }, + {.name = "desc", + .data = N_("Pin domain migrationThreads to host physical CPUs.") + }, + {.name = NULL} +}; + +static const vshCmdOptDef opts_migrationpin[] = { + VIRSH_COMMON_OPT_DOMAIN_FULL(0), + {.name = "cpulist", + .type = VSH_OT_DATA, + .flags = VSH_OFLAG_REQ, + .help = N_("host cpu number(s) to set") + }, + {.name = NULL} +}; + +static bool +cmdMigrationPin(vshControl *ctl, const vshCmd *cmd) +{ + g_autoptr(virshDomain) dom = NULL; + const char *name = NULL; + const char *cpulist = NULL; + int maxcpu; + g_autofree unsigned char *cpumap = NULL; + int cpumaplen; + virshControl *priv = ctl->privData; + + dom = virshCommandOptDomain(ctl, cmd, NULL); + if (!dom) + return false; + + if (vshCommandOptStringReq(ctl, cmd, "domain", &name) < 0) + return false; + + if (vshCommandOptStringReq(ctl, cmd, "cpulist", &cpulist) < 0) + return false; + + maxcpu = virshNodeGetCPUCount(priv->conn); + if (maxcpu < 0) + return false; + + cpumap = virshParseCPUList(ctl, &cpumaplen, cpulist, maxcpu); + if (!cpumap) + return false; + + if (virDomainPinMigrationThread(dom, cpumap, + cpumaplen) != 0) + return false; + + vshPrintExtra(ctl, _("Pin domain %s's migration thread " + "to cpulist %s success.\n"), name, cpulist); + + return true; +} + + const vshCmdDef domManagementCmds[] = { {.name = "attach-device", .handler = cmdAttachDevice, @@ -XXX,XX +XXX,XX @@ const vshCmdDef domManagementCmds[] = { .info = info_dom_fd_associate, .flags = 0 }, + {.name = "migrationpin", + .handler = cmdMigrationPin, + .opts = opts_migrationpin, + .info = info_migrationpin, + .flags = 0 + }, {.name = NULL} }; -- 2.33.0 Add 'migrationpin' to migration parameters to provide another method to set the cpulist required for migration thread pin. 'migrationpin' can only be set before migration and won't effect if 'virsh migrationpin' is set. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- include/libvirt/libvirt-domain.h | 10 ++++++++++ src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_migration.c | 2 ++ src/qemu/qemu_migration.h | 33 ++++++++++++++++---------------- src/qemu/qemu_migration_params.c | 21 ++++++++++++++++++++ src/qemu/qemu_migration_params.h | 5 +++++ 7 files changed, 57 insertions(+), 16 deletions(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index XXXXXXX..XXXXXXX 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -XXX,XX +XXX,XX @@ typedef enum { */ # define VIR_MIGRATE_PARAM_TLS_DESTINATION "tls.destination" +/** + * VIR_MIGRATE_PARAM_MIGRATIONPIN: + * + * virDomainMigrate* params field: the pin of migration threads for + * migration as VIR_TYPED_PARAM_STRING. + * + * Since: 9.1.0 + */ +# define VIR_MIGRATE_PARAM_MIGRATIONPIN "migration.pin" + /* Domain migration. */ virDomainPtr virDomainMigrate (virDomainPtr domain, virConnectPtr dconn, unsigned long flags, const char *dname, diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -XXX,XX +XXX,XX @@ qemuDomainObjPrivateFree(void *data) virObjectUnref(priv->monConfig); g_free(priv->lockState); g_free(priv->origname); + g_free(priv->migrationThreadPinList); virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -XXX,XX +XXX,XX @@ struct _qemuDomainObjPrivate { int nbdPort; /* Port used for migration with NBD */ int migThreadCount; qemuDomainMigThreadInfo **migThreadsInfo; + char *migrationThreadPinList; unsigned short migrationPort; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -XXX,XX +XXX,XX @@ qemuMigrationSrcRun(virQEMUDriver *driver, priv->migMaxBandwidth * 1024 * 1024) < 0) goto error; + qemuMigrationMigrationParamsToVM(migParams, vm); + if (qemuMigrationParamsApply(vm, VIR_ASYNC_JOB_MIGRATION_OUT, migParams, flags) < 0) goto error; diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -XXX,XX +XXX,XX @@ /* All supported migration parameters and their types. */ #define QEMU_MIGRATION_PARAMETERS \ - VIR_MIGRATE_PARAM_URI, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_DEST_NAME, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_DEST_XML, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_BANDWIDTH, VIR_TYPED_PARAM_ULLONG, \ - VIR_MIGRATE_PARAM_GRAPHICS_URI, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_LISTEN_ADDRESS, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_MIGRATE_DISKS, VIR_TYPED_PARAM_STRING | \ - VIR_TYPED_PARAM_MULTIPLE, \ - VIR_MIGRATE_PARAM_DISKS_PORT, VIR_TYPED_PARAM_INT, \ - VIR_MIGRATE_PARAM_COMPRESSION, VIR_TYPED_PARAM_STRING | \ - VIR_TYPED_PARAM_MULTIPLE, \ + VIR_MIGRATE_PARAM_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_DEST_NAME, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_DEST_XML, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_BANDWIDTH, VIR_TYPED_PARAM_ULLONG, \ + VIR_MIGRATE_PARAM_GRAPHICS_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_LISTEN_ADDRESS, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_MIGRATE_DISKS, VIR_TYPED_PARAM_STRING | \ + VIR_TYPED_PARAM_MULTIPLE, \ + VIR_MIGRATE_PARAM_DISKS_PORT, VIR_TYPED_PARAM_INT, \ + VIR_MIGRATE_PARAM_COMPRESSION, VIR_TYPED_PARAM_STRING | \ + VIR_TYPED_PARAM_MULTIPLE, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_LEVEL, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_THREADS, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_DTHREADS, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_XBZRLE_CACHE, VIR_TYPED_PARAM_ULLONG, \ - VIR_MIGRATE_PARAM_PERSIST_XML, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_PERSIST_XML, VIR_TYPED_PARAM_STRING, \ VIR_MIGRATE_PARAM_AUTO_CONVERGE_INITIAL, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_AUTO_CONVERGE_INCREMENT, VIR_TYPED_PARAM_INT, \ - VIR_MIGRATE_PARAM_BANDWIDTH_POSTCOPY, VIR_TYPED_PARAM_ULLONG, \ - VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, VIR_TYPED_PARAM_INT, \ - VIR_MIGRATE_PARAM_TLS_DESTINATION, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_DISKS_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_BANDWIDTH_POSTCOPY, VIR_TYPED_PARAM_ULLONG, \ + VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, VIR_TYPED_PARAM_INT, \ + VIR_MIGRATE_PARAM_TLS_DESTINATION, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_DISKS_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_MIGRATIONPIN, VIR_TYPED_PARAM_STRING, \ NULL diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -XXX,XX +XXX,XX @@ VIR_ENUM_IMPL(qemuMigrationParam, "xbzrle-cache-size", "max-postcopy-bandwidth", "multifd-channels", + "migrationpin", ); typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem; @@ -XXX,XX +XXX,XX @@ static const qemuMigrationParamsTPMapItem qemuMigrationParamsTPMap[] = { {.typedParam = VIR_MIGRATE_PARAM_TLS_DESTINATION, .param = QEMU_MIGRATION_PARAM_TLS_HOSTNAME, .party = QEMU_MIGRATION_SOURCE}, + + {.typedParam = VIR_MIGRATE_PARAM_MIGRATIONPIN, + .param = QEMU_MIGRATION_PARAM_MIGRATIONPIN, + .party = QEMU_MIGRATION_SOURCE}, }; static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = { @@ -XXX,XX +XXX,XX @@ static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = { [QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS] = { .type = QEMU_MIGRATION_PARAM_TYPE_INT, }, + [QEMU_MIGRATION_PARAM_MIGRATIONPIN] = { + .type = QEMU_MIGRATION_PARAM_TYPE_STRING, + }, }; G_STATIC_ASSERT(G_N_ELEMENTS(qemuMigrationParamInfo) == QEMU_MIGRATION_PARAM_LAST); @@ -XXX,XX +XXX,XX @@ qemuMigrationParamsToJSON(qemuMigrationParams *migParams, if (!pv->set) continue; + if (i == QEMU_MIGRATION_PARAM_MIGRATIONPIN) { + continue; + } + if (postcopyResume && !qemuMigrationParamInfo[i].applyOnPostcopyResume) continue; @@ -XXX,XX +XXX,XX @@ qemuMigrationParamsGetMigThreadCount(qemuMigrationParams *migParams, return nmigthreads; } + +void +qemuMigrationMigrationParamsToVM(const qemuMigrationParams *migParams, const virDomainObj *vm) +{ + if (migParams && migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].set) { + qemuDomainObjPrivate *priv = vm->privateData; + priv->migrationThreadPinList = g_strdup(migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].value.s); + } +} diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -XXX,XX +XXX,XX @@ typedef enum { QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE, QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH, QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS, + QEMU_MIGRATION_PARAM_MIGRATIONPIN, QEMU_MIGRATION_PARAM_LAST } qemuMigrationParam; @@ -XXX,XX +XXX,XX @@ qemuMigrationParamsGetTLSHostname(qemuMigrationParams *migParams); int qemuMigrationParamsGetMigThreadCount(qemuMigrationParams *migParams, unsigned long migflags); + +void +qemuMigrationMigrationParamsToVM(const qemuMigrationParams *migParams, + const virDomainObj *vm); -- 2.33.0 If pcpumap is set before migration, pin migration thread to expected cpulist immediately after get thread info during migration. Add 'qemuProcessGetPcpumap' to get pcpumap, try get the result of 'virsh migratepin' first, and if it's not set, try to get the pcpumap from migration parameter. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_migration.c | 28 ++++++++++++++ src/qemu/qemu_process.c | 81 +++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 3 ++ 3 files changed, 112 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -XXX,XX +XXX,XX @@ qemuMigrationAnyCompleted(virDomainObj *vm, return -1; } + +static void +qemuMigrationSrcThreadPin(virDomainObj *vm, + virBitmap *pcpumap, + int nmigthreads) +{ + qemuDomainObjPrivate *priv = vm->privateData; + qemuDomainMigThreadInfo **migthreads = priv->migThreadsInfo; + size_t i; + + + for (i = 0; i < nmigthreads; i++) { + qemuProcessSetMigThreadAffinity(priv, vm, migthreads[i]->thread_id, pcpumap); + } +} + + /* Returns 0 on get migration thread info success, * -1 on unexpected failure */ @@ -XXX,XX +XXX,XX @@ qemuMigrationSrcWaitForThreadInfo(virDomainObj *vm, qemuDomainObjPrivate *priv = vm->privateData; qemuMonitorMigThreadInfo **infolist = NULL; qemuDomainMigThreadInfo **migthreads = NULL; + virBitmap *pcpumap = NULL; int nmigthreads = 0; int rc; size_t i; @@ -XXX,XX +XXX,XX @@ qemuMigrationSrcWaitForThreadInfo(virDomainObj *vm, priv->migThreadsInfo = migthreads; ret = 0; + /* Try get pcpumap for migration thread pin, + * but thread pin failed shouldn't effect migration */ + if (!(pcpumap = qemuProcessGetPcpumap(priv))) + goto cleanup; + + qemuMigrationSrcThreadPin(vm, pcpumap, nmigthreads); + cleanup: + if (pcpumap != priv->pcpumap) + virBitmapFree(pcpumap); + if (migthreads && (ret < 0)) { qemuDomainMigThreadsInfoFree(migthreads, priv->migThreadCount); priv->migThreadCount = 0; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv, virDomainMigrationIDDefFree(migration); return; } + + +static unsigned char * +virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu) +{ + int lastcpu; + unsigned char *cpumap = NULL; + virBitmap *map = NULL; + + if (cpulist[0] == 'r') { + map = virBitmapNew(maxcpu); + if (!map) + return NULL; + virBitmapSetAll(map); + } else { + if (virBitmapParse(cpulist, &map, 1024) < 0 || + virBitmapIsAllClear(map)) { + goto cleanup; + } + + lastcpu = virBitmapLastSetBit(map); + if (lastcpu >= maxcpu) + goto cleanup; + } + + if (virBitmapToData(map, &cpumap, cpumaplen) < 0) + VIR_ERROR(_("Bitmap to data failure")); + + cleanup: + virBitmapFree(map); + return cpumap; +} + + +/* + * The value of "virsh migrationpin" is saved to priv->pcpumap + * If priv->pcpumap is NULL, it means migrationpin command is not called, + * otherwise we set the affinity of migration thread by migrationpin + */ +virBitmap* +qemuProcessGetPcpumap(qemuDomainObjPrivate *priv) +{ + int cpumaplen = 0; + int maxcpu = 0; + virBitmap *pcpumap = NULL; + g_autofree unsigned char *cpumap = NULL; + + /* priv->pcpumap = NULL means migrationpin is not set, try to get pcpumap + * from migration parameter. */ + if (priv->pcpumap) + return priv->pcpumap; + + if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) { + VIR_ERROR(_("didn't set the migratin thread pin")); + return NULL; + } + + /* judge whether set_migration_pin is default value or not */ + if (STREQ(priv->migrationThreadPinList, "none")) + return NULL; + + maxcpu = virHostCPUGetCount(); + if (maxcpu < 0) { + VIR_ERROR(_("get the cpu count of host failure")); + return NULL; + } + + cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu); + if (!cpumap) { + VIR_ERROR(_("parse migration.pin param failure : migration.pin = %s"), + priv->migrationThreadPinList); + return NULL; + } + + if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) { + VIR_ERROR(_("Bitmap data failure")); + return NULL; + } + + return pcpumap; +} diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -XXX,XX +XXX,XX @@ qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv, virDomainObj *vm, int mpid, virBitmap *pcpumap); + +virBitmap* +qemuProcessGetPcpumap(qemuDomainObjPrivate *priv); -- 2.33.0 Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_process.c | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index XXXXXXX..XXXXXXX 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -XXX,XX +XXX,XX @@ qemuProcessHandleSpiceMigrated(qemuMonitor *mon G_GNUC_UNUSED, } +static void +qemuProcessHandleMigrationPinStatus(virDomainObj *vm, int status) +{ + qemuDomainObjPrivate *priv = vm->privateData; + + if (vm->job->asyncJob != VIR_ASYNC_JOB_MIGRATION_OUT) + return; + + switch (status) { + case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_SETUP: + case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER: + case QEMU_MONITOR_MIGRATION_STATUS_DEVICE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY_PAUSED: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY_RECOVER: + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: + case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED: + case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG: + break; + case QEMU_MONITOR_MIGRATION_STATUS_ERROR: + /* + * migration thread is still running, + * so we can't delete migration Cgroup. + */ + qemuDomainMigThreadsInfoFree(priv->migThreadsInfo, priv->migThreadCount); + priv->migThreadsInfo = NULL; + priv->migThreadCount = 0; + g_free(priv->migrationThreadPinList); + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + break; + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED: + qemuDomainMigThreadsInfoFree(priv->migThreadsInfo, priv->migThreadCount); + priv->migThreadsInfo = NULL; + priv->migThreadCount = 0; + g_free(priv->migrationThreadPinList); + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + if (virCgroupDelThread(priv->cgroup, + VIR_CGROUP_THREAD_MIGRATION_THREAD, 0) < 0) + VIR_WARN("Failed to delete migration thread Cgroup!"); + VIR_INFO("success to free pcpumap and migrationPids"); + break; + default: + VIR_WARN("got unknown migration status'%s'", + qemuMonitorMigrationStatusTypeToString(status)); + break; + } + + return; +} + + static void qemuProcessHandleMigrationStatus(qemuMonitor *mon G_GNUC_UNUSED, virDomainObj *vm, @@ -XXX,XX +XXX,XX @@ qemuProcessHandleMigrationStatus(qemuMonitor *mon G_GNUC_UNUSED, default: break; } + qemuProcessHandleMigrationPinStatus(vm, status); cleanup: virObjectUnlock(vm); -- 2.33.0