From nobody Tue Apr 30 03:06:01 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=libvir-list-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=libvir-list-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1548857340502114.17786373370757; Wed, 30 Jan 2019 06:09:00 -0800 (PST) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.14]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id A6BA1C0CD175; Wed, 30 Jan 2019 14:08:57 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.20]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 87E2E5D97A; Wed, 30 Jan 2019 14:08:56 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 8898118033CA; Wed, 30 Jan 2019 14:08:54 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.phx2.redhat.com [10.5.11.13]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id x0UDv7r5025070 for ; Wed, 30 Jan 2019 08:57:07 -0500 Received: by smtp.corp.redhat.com (Postfix) id 0CC0D608C6; Wed, 30 Jan 2019 13:57:07 +0000 (UTC) Received: from moe.brq.redhat.com (unknown [10.43.2.192]) by smtp.corp.redhat.com (Postfix) with ESMTP id 82BD0608C4 for ; Wed, 30 Jan 2019 13:57:01 +0000 (UTC) From: Michal Privoznik To: libvir-list@redhat.com Date: Wed, 30 Jan 2019 14:56:46 +0100 Message-Id: <9a4ca2d2a5be06e1366bb67104070996ea3868bd.1548856606.git.mprivozn@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.79 on 10.5.11.13 X-loop: libvir-list@redhat.com Subject: [libvirt] [PATCH] qemu: Rework setting process affinity X-BeenThere: libvir-list@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Development discussions about the libvirt library & tools List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Content-Transfer-Encoding: quoted-printable Sender: libvir-list-bounces@redhat.com Errors-To: libvir-list-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.14 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.32]); Wed, 30 Jan 2019 14:08:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" https://bugzilla.redhat.com/show_bug.cgi?id=3D1503284 The way we currently start qemu from CPU affinity POV is as follows: 1) the child process is set affinity to all online CPUs (unless some vcpu pinning was given in the domain XML) 2) Once qemu is running, cpuset cgroup is configured taking memory pinning into account Problem is that we let qemu allocate its memory just anywhere in 1) and then rely in 2) to be able to move the memory to configured NUMA nodes. This might not be always possible (e.g. qemu might lock some parts of its memory) and is very suboptimal (copying large memory between NUMA nodes takes significant amount of time). The solution is to set the affinity correctly from the beginning and then possibly refine it later via cgroups. Signed-off-by: Michal Privoznik --- src/qemu/qemu_process.c | 152 ++++++++++++++++++++++------------------ 1 file changed, 83 insertions(+), 69 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 9ccc3601a2..a4668f6773 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -2435,6 +2435,44 @@ qemuProcessDetectIOThreadPIDs(virQEMUDriverPtr drive= r, } =20 =20 +static int +qemuProcessGetAllCpuAffinity(pid_t pid, + virBitmapPtr *cpumapRet) +{ + VIR_AUTOPTR(virBitmap) cpumap =3D NULL; + VIR_AUTOPTR(virBitmap) hostcpumap =3D NULL; + int hostcpus; + + *cpumapRet =3D NULL; + + if (!virHostCPUHasBitmap()) + return 0; + + if (!(hostcpumap =3D virHostCPUGetOnlineBitmap()) || + !(cpumap =3D virProcessGetAffinity(pid))) + return -1; + + if (!virBitmapEqual(hostcpumap, cpumap)) { + /* setaffinity fails if you set bits for CPUs which + * aren't present, so we have to limit ourselves */ + if ((hostcpus =3D virHostCPUGetCount()) < 0) + return -1; + + if (hostcpus > QEMUD_CPUMASK_LEN) + hostcpus =3D QEMUD_CPUMASK_LEN; + + virBitmapFree(cpumap); + if (!(cpumap =3D virBitmapNew(hostcpus))) + return -1; + + virBitmapSetAll(cpumap); + } + + VIR_STEAL_PTR(*cpumapRet, cpumap); + return 0; +} + + /* * To be run between fork/exec of QEMU only */ @@ -2443,9 +2481,9 @@ static int qemuProcessInitCpuAffinity(virDomainObjPtr vm) { int ret =3D -1; - virBitmapPtr cpumap =3D NULL; virBitmapPtr cpumapToSet =3D NULL; - virBitmapPtr hostcpumap =3D NULL; + VIR_AUTOPTR(virBitmap) hostcpumap =3D NULL; + virDomainNumatuneMemMode mem_mode; qemuDomainObjPrivatePtr priv =3D vm->privateData; =20 if (!vm->pid) { @@ -2454,59 +2492,36 @@ qemuProcessInitCpuAffinity(virDomainObjPtr vm) return -1; } =20 - if (vm->def->placement_mode =3D=3D VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)= { - VIR_DEBUG("Set CPU affinity with advisory nodeset from numad"); - cpumapToSet =3D priv->autoCpuset; + /* Here is the deal, we can't set cpuset.mems before qemu is + * started as it clashes with KVM allocation. Therefore we + * used to let qemu allocate its memory anywhere as we would + * then move the memory to desired NUMA node via CGroups. + * However, that might not be always possible because qemu + * might lock some parts of its memory (e.g. due to VFIO). + * Solution is to set some temporary affinity now and then + * fix it later, once qemu is already running. */ + if (virDomainNumaGetNodeCount(vm->def->numa) <=3D 1 && + virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) =3D=3D 0 && + mem_mode =3D=3D VIR_DOMAIN_NUMATUNE_MEM_STRICT) { + if (virDomainNumatuneMaybeGetNodeset(vm->def->numa, + priv->autoNodeset, + &cpumapToSet, + -1) < 0) + goto cleanup; + } else if (vm->def->cputune.emulatorpin) { + cpumapToSet =3D vm->def->cputune.emulatorpin; } else { - VIR_DEBUG("Set CPU affinity with specified cpuset"); - if (vm->def->cpumask) { - cpumapToSet =3D vm->def->cpumask; - } else { - /* You may think this is redundant, but we can't assume libvir= td - * itself is running on all pCPUs, so we need to explicitly set - * the spawned QEMU instance to all pCPUs if no map is given in - * its config file */ - int hostcpus; - - if (virHostCPUHasBitmap()) { - hostcpumap =3D virHostCPUGetOnlineBitmap(); - cpumap =3D virProcessGetAffinity(vm->pid); - } - - if (hostcpumap && cpumap && virBitmapEqual(hostcpumap, cpumap)= ) { - /* we're using all available CPUs, no reason to set - * mask. If libvirtd is running without explicit - * affinity, we can use hotplugged CPUs for this VM */ - ret =3D 0; - goto cleanup; - } else { - /* setaffinity fails if you set bits for CPUs which - * aren't present, so we have to limit ourselves */ - if ((hostcpus =3D virHostCPUGetCount()) < 0) - goto cleanup; - - if (hostcpus > QEMUD_CPUMASK_LEN) - hostcpus =3D QEMUD_CPUMASK_LEN; - - virBitmapFree(cpumap); - if (!(cpumap =3D virBitmapNew(hostcpus))) - goto cleanup; - - virBitmapSetAll(cpumap); - - cpumapToSet =3D cpumap; - } - } + if (qemuProcessGetAllCpuAffinity(vm->pid, &hostcpumap) < 0) + goto cleanup; + cpumapToSet =3D hostcpumap; } =20 - if (virProcessSetAffinity(vm->pid, cpumapToSet) < 0) + if (cpumapToSet && + virProcessSetAffinity(vm->pid, cpumapToSet) < 0) goto cleanup; =20 ret =3D 0; - cleanup: - virBitmapFree(cpumap); - virBitmapFree(hostcpumap); return ret; } #else /* !defined(HAVE_SCHED_GETAFFINITY) && !defined(HAVE_BSD_CPU_AFFINIT= Y) */ @@ -2586,7 +2601,8 @@ qemuProcessSetupPid(virDomainObjPtr vm, qemuDomainObjPrivatePtr priv =3D vm->privateData; virDomainNumatuneMemMode mem_mode; virCgroupPtr cgroup =3D NULL; - virBitmapPtr use_cpumask; + virBitmapPtr use_cpumask =3D NULL; + VIR_AUTOPTR(virBitmap) hostcpumap =3D NULL; char *mem_mask =3D NULL; int ret =3D -1; =20 @@ -2598,12 +2614,21 @@ qemuProcessSetupPid(virDomainObjPtr vm, } =20 /* Infer which cpumask shall be used. */ - if (cpumask) + if (cpumask) { use_cpumask =3D cpumask; - else if (vm->def->placement_mode =3D=3D VIR_DOMAIN_CPU_PLACEMENT_MODE_= AUTO) + } else if (vm->def->placement_mode =3D=3D VIR_DOMAIN_CPU_PLACEMENT_MOD= E_AUTO) { use_cpumask =3D priv->autoCpuset; - else + } else if (vm->def->cpumask) { use_cpumask =3D vm->def->cpumask; + } else if (virHostCPUHasBitmap()) { + /* You may think this is redundant, but we can't assume libvirtd + * itself is running on all pCPUs, so we need to explicitly set + * the spawned QEMU instance to all pCPUs if no map is given in + * its config file */ + if (qemuProcessGetAllCpuAffinity(pid, &hostcpumap) < 0) + goto cleanup; + use_cpumask =3D hostcpumap; + } =20 /* * If CPU cgroup controller is not initialized here, then we need @@ -2628,13 +2653,7 @@ qemuProcessSetupPid(virDomainObjPtr vm, qemuSetupCgroupCpusetCpus(cgroup, use_cpumask) < 0) goto cleanup; =20 - /* - * Don't setup cpuset.mems for the emulator, they need to - * be set up after initialization in order for kvm - * allocations to succeed. - */ - if (nameval !=3D VIR_CGROUP_THREAD_EMULATOR && - mem_mask && virCgroupSetCpusetMems(cgroup, mem_mask) < 0) + if (mem_mask && virCgroupSetCpusetMems(cgroup, mem_mask) < 0) goto cleanup; =20 } @@ -6634,12 +6653,7 @@ qemuProcessLaunch(virConnectPtr conn, =20 /* This must be done after cgroup placement to avoid resetting CPU * affinity */ - if (!vm->def->cputune.emulatorpin && - qemuProcessInitCpuAffinity(vm) < 0) - goto cleanup; - - VIR_DEBUG("Setting emulator tuning/settings"); - if (qemuProcessSetupEmulator(vm) < 0) + if (qemuProcessInitCpuAffinity(vm) < 0) goto cleanup; =20 VIR_DEBUG("Setting cgroup for external devices (if required)"); @@ -6708,10 +6722,6 @@ qemuProcessLaunch(virConnectPtr conn, if (qemuProcessUpdateAndVerifyCPU(driver, vm, asyncJob) < 0) goto cleanup; =20 - VIR_DEBUG("Setting up post-init cgroup restrictions"); - if (qemuSetupCpusetMems(vm) < 0) - goto cleanup; - VIR_DEBUG("setting up hotpluggable cpus"); if (qemuDomainHasHotpluggableStartupVcpus(vm->def)) { if (qemuDomainRefreshVcpuInfo(driver, vm, asyncJob, false) < 0) @@ -6737,6 +6747,10 @@ qemuProcessLaunch(virConnectPtr conn, if (qemuProcessDetectIOThreadPIDs(driver, vm, asyncJob) < 0) goto cleanup; =20 + VIR_DEBUG("Setting emulator tuning/settings"); + if (qemuProcessSetupEmulator(vm) < 0) + goto cleanup; + VIR_DEBUG("Setting global CPU cgroup (if required)"); if (qemuSetupGlobalCpuCgroup(vm) < 0) goto cleanup; --=20 2.19.2 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list