[PATCH v3 2/2] KVM: selftests: Test behavior of KVM_X86_DISABLE_EXITS_APERFMPERF

Jim Mattson posted 2 patches 10 months, 3 weeks ago
There is a newer version of this series
[PATCH v3 2/2] KVM: selftests: Test behavior of KVM_X86_DISABLE_EXITS_APERFMPERF
Posted by Jim Mattson 10 months, 3 weeks ago
For a VCPU thread pinned to a single LPU, verify that interleaved host
and guest reads of IA32_[AM]PERF return strictly increasing values when
APERFMPERF exiting is disabled.

Signed-off-by: Jim Mattson <jmattson@google.com>
---
 tools/testing/selftests/kvm/Makefile.kvm      |   1 +
 .../selftests/kvm/x86/aperfmperf_test.c       | 162 ++++++++++++++++++
 2 files changed, 163 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/x86/aperfmperf_test.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 4277b983cace..bfee69b33310 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -116,6 +116,7 @@ TEST_GEN_PROGS_x86 += x86/amx_test
 TEST_GEN_PROGS_x86 += x86/max_vcpuid_cap_test
 TEST_GEN_PROGS_x86 += x86/triple_fault_event_test
 TEST_GEN_PROGS_x86 += x86/recalc_apic_map_test
+TEST_GEN_PROGS_x86 += x86/aperfmperf_test
 TEST_GEN_PROGS_x86 += access_tracking_perf_test
 TEST_GEN_PROGS_x86 += coalesced_io_test
 TEST_GEN_PROGS_x86 += demand_paging_test
diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c
new file mode 100644
index 000000000000..7473afb7f6fa
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Test for KVM_X86_DISABLE_EXITS_APERFMPERF
+ *
+ * Copyright (C) 2025, Google LLC.
+ *
+ * Test the ability to disable VM-exits for rdmsr of IA32_APERF and
+ * IA32_MPERF. When these VM-exits are disabled, reads of these MSRs
+ * return the host's values.
+ *
+ * Note: Requires read access to /dev/cpu/<lpu>/msr to read host MSRs.
+ */
+
+#include <fcntl.h>
+#include <limits.h>
+#include <pthread.h>
+#include <sched.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <asm/msr-index.h>
+
+#include "kvm_util.h"
+#include "processor.h"
+#include "test_util.h"
+
+#define NUM_ITERATIONS 100
+
+static void pin_thread(int cpu)
+{
+	cpu_set_t cpuset;
+	int rc;
+
+	CPU_ZERO(&cpuset);
+	CPU_SET(cpu, &cpuset);
+
+	rc = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
+	TEST_ASSERT(rc == 0, "%s: Can't set thread affinity", __func__);
+}
+
+static int open_dev_msr(int cpu)
+{
+	char path[PATH_MAX];
+	int msr_fd;
+
+	snprintf(path, sizeof(path), "/dev/cpu/%d/msr", cpu);
+	msr_fd = open(path, O_RDONLY);
+	__TEST_REQUIRE(msr_fd >= 0, "Can't open %s for read", path);
+
+	return msr_fd;
+}
+
+static uint64_t read_dev_msr(int msr_fd, uint32_t msr)
+{
+	uint64_t data;
+	ssize_t rc;
+
+	rc = pread(msr_fd, &data, sizeof(data), msr);
+	TEST_ASSERT(rc == sizeof(data), "Read of MSR 0x%x failed", msr);
+
+	return data;
+}
+
+static void guest_code(void)
+{
+	int i;
+
+	for (i = 0; i < NUM_ITERATIONS; i++) {
+		uint64_t aperf = rdmsr(MSR_IA32_APERF);
+		uint64_t mperf = rdmsr(MSR_IA32_MPERF);
+
+		GUEST_SYNC2(aperf, mperf);
+	}
+
+	GUEST_DONE();
+}
+
+static bool kvm_can_disable_aperfmperf_exits(struct kvm_vm *vm)
+{
+	int flags = vm_check_cap(vm, KVM_CAP_X86_DISABLE_EXITS);
+
+	return flags & KVM_X86_DISABLE_EXITS_APERFMPERF;
+}
+
+int main(int argc, char *argv[])
+{
+	uint64_t host_aperf_before, host_mperf_before;
+	int cpu = sched_getcpu();
+	struct kvm_vcpu *vcpu;
+	struct kvm_vm *vm;
+	int msr_fd;
+	int i;
+
+	pin_thread(cpu);
+
+	msr_fd = open_dev_msr(cpu);
+
+	/*
+	 * This test requires a non-standard VM initialization, because
+	 * KVM_ENABLE_CAP cannot be used on a VM file descriptor after
+	 * a VCPU has been created.
+	 */
+	vm = vm_create(1);
+
+	TEST_REQUIRE(kvm_can_disable_aperfmperf_exits(vm));
+
+	vm_enable_cap(vm, KVM_CAP_X86_DISABLE_EXITS,
+		      KVM_X86_DISABLE_EXITS_APERFMPERF);
+
+	vcpu = vm_vcpu_add(vm, 0, guest_code);
+
+	host_aperf_before = read_dev_msr(msr_fd, MSR_IA32_APERF);
+	host_mperf_before = read_dev_msr(msr_fd, MSR_IA32_MPERF);
+
+	for (i = 0; i < NUM_ITERATIONS; i++) {
+		uint64_t host_aperf_after, host_mperf_after;
+		uint64_t guest_aperf, guest_mperf;
+		struct ucall uc;
+
+		vcpu_run(vcpu);
+		TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
+
+		switch (get_ucall(vcpu, &uc)) {
+		case UCALL_DONE:
+			break;
+		case UCALL_ABORT:
+			REPORT_GUEST_ASSERT(uc);
+		case UCALL_SYNC:
+			guest_aperf = uc.args[0];
+			guest_mperf = uc.args[1];
+
+			host_aperf_after = read_dev_msr(msr_fd, MSR_IA32_APERF);
+			host_mperf_after = read_dev_msr(msr_fd, MSR_IA32_MPERF);
+
+			TEST_ASSERT(host_aperf_before < guest_aperf,
+				    "APERF: host_before (%lu) >= guest (%lu)",
+				    host_aperf_before, guest_aperf);
+			TEST_ASSERT(guest_aperf < host_aperf_after,
+				    "APERF: guest (%lu) >= host_after (%lu)",
+				    guest_aperf, host_aperf_after);
+			TEST_ASSERT(host_mperf_before < guest_mperf,
+				    "MPERF: host_before (%lu) >= guest (%lu)",
+				    host_mperf_before, guest_mperf);
+			TEST_ASSERT(guest_mperf < host_mperf_after,
+				    "MPERF: guest (%lu) >= host_after (%lu)",
+				    guest_mperf, host_mperf_after);
+
+			host_aperf_before = host_aperf_after;
+			host_mperf_before = host_mperf_after;
+
+			break;
+		}
+	}
+
+	TEST_ASSERT_EQ(i, NUM_ITERATIONS);
+
+	kvm_vm_free(vm);
+	close(msr_fd);
+
+	return 0;
+}
-- 
2.49.0.395.g12beb8f557-goog
Re: [PATCH v3 2/2] KVM: selftests: Test behavior of KVM_X86_DISABLE_EXITS_APERFMPERF
Posted by Sean Christopherson 9 months, 2 weeks ago
On Fri, Mar 21, 2025, Jim Mattson wrote:
> +#include <fcntl.h>
> +#include <limits.h>
> +#include <pthread.h>
> +#include <sched.h>
> +#include <stdbool.h>
> +#include <stdio.h>
> +#include <stdint.h>
> +#include <unistd.h>
> +#include <asm/msr-index.h>
> +
> +#include "kvm_util.h"
> +#include "processor.h"
> +#include "test_util.h"
> +
> +#define NUM_ITERATIONS 100
> +
> +static void pin_thread(int cpu)
> +{
> +	cpu_set_t cpuset;
> +	int rc;
> +
> +	CPU_ZERO(&cpuset);
> +	CPU_SET(cpu, &cpuset);
> +
> +	rc = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
> +	TEST_ASSERT(rc == 0, "%s: Can't set thread affinity", __func__);

Heh, you copy-pasted this from hardware_disable_test.c, didn't you?  :-)

Would it make sense to turn this into a generic API that takes care of the entire
sched_getcpu() => pthread_setaffinity_np()?  E.g. kvm_pin_task_to_current_cpu().
I suspect there are other (potential) tests that don't care about what CPU they
run on, so long as the test is pinned.

> +}
> +
> +static int open_dev_msr(int cpu)
> +{
> +	char path[PATH_MAX];
> +	int msr_fd;
> +
> +	snprintf(path, sizeof(path), "/dev/cpu/%d/msr", cpu);
> +	msr_fd = open(path, O_RDONLY);
> +	__TEST_REQUIRE(msr_fd >= 0, "Can't open %s for read", path);

Please use open_path_or_exit().

Hmm, and I'm planning on posting a small series to add a variant that takes an
ENOENT message, and spits out a (hopefully) helpful message for the EACCES case.
It would be nice to have this one spit out something like "Is msk.ko loaded?",
but I would say don't worry about trying to coordinate anything.  Worst case
scenario we can add a help message when the dust settles.

> +	return msr_fd;
> +}
> +
> +static uint64_t read_dev_msr(int msr_fd, uint32_t msr)
> +{
> +	uint64_t data;
> +	ssize_t rc;
> +
> +	rc = pread(msr_fd, &data, sizeof(data), msr);
> +	TEST_ASSERT(rc == sizeof(data), "Read of MSR 0x%x failed", msr);
> +
> +	return data;
> +}
> +
> +static void guest_code(void)
> +{
> +	int i;
> +
> +	for (i = 0; i < NUM_ITERATIONS; i++) {
> +		uint64_t aperf = rdmsr(MSR_IA32_APERF);
> +		uint64_t mperf = rdmsr(MSR_IA32_MPERF);
> +
> +		GUEST_SYNC2(aperf, mperf);

Does the test generate multiple RDMSR per MSR if you do:

		GUEST_SYNC2(rdmsr(MSR_IA32_APERF), rdmsr(MSR_IA32_MPERF));

If the code generation comes out


> +	}
> +
> +	GUEST_DONE();
> +}
> +
> +static bool kvm_can_disable_aperfmperf_exits(struct kvm_vm *vm)
> +{
> +	int flags = vm_check_cap(vm, KVM_CAP_X86_DISABLE_EXITS);
> +
> +	return flags & KVM_X86_DISABLE_EXITS_APERFMPERF;
> +}

Please don't add one-off helpers like this, especially when they're the condition
for TEST_REQUIRE().  I *want* the gory details if the test is skipped, so that I
don't have to go look at the source code to figure out what's missing.

And it's literally more code.

> +
> +int main(int argc, char *argv[])
> +{
> +	uint64_t host_aperf_before, host_mperf_before;
> +	int cpu = sched_getcpu();
> +	struct kvm_vcpu *vcpu;
> +	struct kvm_vm *vm;
> +	int msr_fd;
> +	int i;
> +
> +	pin_thread(cpu);
> +
> +	msr_fd = open_dev_msr(cpu);
> +
> +	/*
> +	 * This test requires a non-standard VM initialization, because
> +	 * KVM_ENABLE_CAP cannot be used on a VM file descriptor after
> +	 * a VCPU has been created.

Hrm, we should really sort this out.  Every test that needs to enable a capability
is having to copy+paste this pattern.  I don't love the idea of expanding
__vm_create_with_one_vcpu(), but there's gotta be a solution that isn't horrible,
and anything is better than endly copy paste.

> +	 */
> +	vm = vm_create(1);
> +
> +	TEST_REQUIRE(kvm_can_disable_aperfmperf_exits(vm));

	TEST_REQUIRE(vm_check_cap(vm, KVM_CAP_X86_DISABLE_EXITS) &
		     KVM_X86_DISABLE_EXITS_APERFMPERF);
> +
> +	vm_enable_cap(vm, KVM_CAP_X86_DISABLE_EXITS,
> +		      KVM_X86_DISABLE_EXITS_APERFMPERF);
> +
> +	vcpu = vm_vcpu_add(vm, 0, guest_code);
> +
> +	host_aperf_before = read_dev_msr(msr_fd, MSR_IA32_APERF);
> +	host_mperf_before = read_dev_msr(msr_fd, MSR_IA32_MPERF);
> +
> +	for (i = 0; i < NUM_ITERATIONS; i++) {
> +		uint64_t host_aperf_after, host_mperf_after;
> +		uint64_t guest_aperf, guest_mperf;
> +		struct ucall uc;
> +
> +		vcpu_run(vcpu);
> +		TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
> +
> +		switch (get_ucall(vcpu, &uc)) {
> +		case UCALL_DONE:
> +			break;
> +		case UCALL_ABORT:
> +			REPORT_GUEST_ASSERT(uc);
> +		case UCALL_SYNC:
> +			guest_aperf = uc.args[0];
> +			guest_mperf = uc.args[1];
> +
> +			host_aperf_after = read_dev_msr(msr_fd, MSR_IA32_APERF);
> +			host_mperf_after = read_dev_msr(msr_fd, MSR_IA32_MPERF);
> +
> +			TEST_ASSERT(host_aperf_before < guest_aperf,
> +				    "APERF: host_before (%lu) >= guest (%lu)",
> +				    host_aperf_before, guest_aperf);

Honest question, is decimal really better than hex for these?

> +			TEST_ASSERT(guest_aperf < host_aperf_after,
> +				    "APERF: guest (%lu) >= host_after (%lu)",
> +				    guest_aperf, host_aperf_after);
> +			TEST_ASSERT(host_mperf_before < guest_mperf,
> +				    "MPERF: host_before (%lu) >= guest (%lu)",
> +				    host_mperf_before, guest_mperf);
> +			TEST_ASSERT(guest_mperf < host_mperf_after,
> +				    "MPERF: guest (%lu) >= host_after (%lu)",
> +				    guest_mperf, host_mperf_after);
> +
> +			host_aperf_before = host_aperf_after;
> +			host_mperf_before = host_mperf_after;
> +
> +			break;
> +		}
> +	}
> +
> +	TEST_ASSERT_EQ(i, NUM_ITERATIONS);

Why?
Re: [PATCH v3 2/2] KVM: selftests: Test behavior of KVM_X86_DISABLE_EXITS_APERFMPERF
Posted by Jim Mattson 9 months, 1 week ago
On Mon, Apr 28, 2025 at 6:26 PM Sean Christopherson <seanjc@google.com> wrote:
>
> On Fri, Mar 21, 2025, Jim Mattson wrote:
> > +#include <fcntl.h>
> > +#include <limits.h>
> > +#include <pthread.h>
> > +#include <sched.h>
> > +#include <stdbool.h>
> > +#include <stdio.h>
> > +#include <stdint.h>
> > +#include <unistd.h>
> > +#include <asm/msr-index.h>
> > +
> > +#include "kvm_util.h"
> > +#include "processor.h"
> > +#include "test_util.h"
> > +
> > +#define NUM_ITERATIONS 100
> > +
> > +static void pin_thread(int cpu)
> > +{
> > +     cpu_set_t cpuset;
> > +     int rc;
> > +
> > +     CPU_ZERO(&cpuset);
> > +     CPU_SET(cpu, &cpuset);
> > +
> > +     rc = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
> > +     TEST_ASSERT(rc == 0, "%s: Can't set thread affinity", __func__);
>
> Heh, you copy-pasted this from hardware_disable_test.c, didn't you?  :-)

Probably.

> Would it make sense to turn this into a generic API that takes care of the entire
> sched_getcpu() => pthread_setaffinity_np()?  E.g. kvm_pin_task_to_current_cpu().
> I suspect there are other (potential) tests that don't care about what CPU they
> run on, so long as the test is pinned.

Sure.

> > +}
> > +
> > +static int open_dev_msr(int cpu)
> > +{
> > +     char path[PATH_MAX];
> > +     int msr_fd;
> > +
> > +     snprintf(path, sizeof(path), "/dev/cpu/%d/msr", cpu);
> > +     msr_fd = open(path, O_RDONLY);
> > +     __TEST_REQUIRE(msr_fd >= 0, "Can't open %s for read", path);
>
> Please use open_path_or_exit().

TIL.

> Hmm, and I'm planning on posting a small series to add a variant that takes an
> ENOENT message, and spits out a (hopefully) helpful message for the EACCES case.
> It would be nice to have this one spit out something like "Is msk.ko loaded?",
> but I would say don't worry about trying to coordinate anything.  Worst case
> scenario we can add a help message when the dust settles.
>
> > +     return msr_fd;
> > +}
> > +
> > +static uint64_t read_dev_msr(int msr_fd, uint32_t msr)
> > +{
> > +     uint64_t data;
> > +     ssize_t rc;
> > +
> > +     rc = pread(msr_fd, &data, sizeof(data), msr);
> > +     TEST_ASSERT(rc == sizeof(data), "Read of MSR 0x%x failed", msr);
> > +
> > +     return data;
> > +}
> > +
> > +static void guest_code(void)
> > +{
> > +     int i;
> > +
> > +     for (i = 0; i < NUM_ITERATIONS; i++) {
> > +             uint64_t aperf = rdmsr(MSR_IA32_APERF);
> > +             uint64_t mperf = rdmsr(MSR_IA32_MPERF);
> > +
> > +             GUEST_SYNC2(aperf, mperf);
>
> Does the test generate multiple RDMSR per MSR if you do:
>
>                 GUEST_SYNC2(rdmsr(MSR_IA32_APERF), rdmsr(MSR_IA32_MPERF));
>
> If the code generation comes out

I'll have to check.

>
> > +     }
> > +
> > +     GUEST_DONE();
> > +}
> > +
> > +static bool kvm_can_disable_aperfmperf_exits(struct kvm_vm *vm)
> > +{
> > +     int flags = vm_check_cap(vm, KVM_CAP_X86_DISABLE_EXITS);
> > +
> > +     return flags & KVM_X86_DISABLE_EXITS_APERFMPERF;
> > +}
>
> Please don't add one-off helpers like this, especially when they're the condition
> for TEST_REQUIRE().  I *want* the gory details if the test is skipped, so that I
> don't have to go look at the source code to figure out what's missing.
>
> And it's literally more code.

Okay.

> > +
> > +int main(int argc, char *argv[])
> > +{
> > +     uint64_t host_aperf_before, host_mperf_before;
> > +     int cpu = sched_getcpu();
> > +     struct kvm_vcpu *vcpu;
> > +     struct kvm_vm *vm;
> > +     int msr_fd;
> > +     int i;
> > +
> > +     pin_thread(cpu);
> > +
> > +     msr_fd = open_dev_msr(cpu);
> > +
> > +     /*
> > +      * This test requires a non-standard VM initialization, because
> > +      * KVM_ENABLE_CAP cannot be used on a VM file descriptor after
> > +      * a VCPU has been created.
>
> Hrm, we should really sort this out.  Every test that needs to enable a capability
> is having to copy+paste this pattern.  I don't love the idea of expanding
> __vm_create_with_one_vcpu(), but there's gotta be a solution that isn't horrible,
> and anything is better than endly copy paste.

This is all your fault, I believe. But, I'll see what I can do.

> > +      */
> > +     vm = vm_create(1);
> > +
> > +     TEST_REQUIRE(kvm_can_disable_aperfmperf_exits(vm));
>
>         TEST_REQUIRE(vm_check_cap(vm, KVM_CAP_X86_DISABLE_EXITS) &
>                      KVM_X86_DISABLE_EXITS_APERFMPERF);
> > +
> > +     vm_enable_cap(vm, KVM_CAP_X86_DISABLE_EXITS,
> > +                   KVM_X86_DISABLE_EXITS_APERFMPERF);
> > +
> > +     vcpu = vm_vcpu_add(vm, 0, guest_code);
> > +
> > +     host_aperf_before = read_dev_msr(msr_fd, MSR_IA32_APERF);
> > +     host_mperf_before = read_dev_msr(msr_fd, MSR_IA32_MPERF);
> > +
> > +     for (i = 0; i < NUM_ITERATIONS; i++) {
> > +             uint64_t host_aperf_after, host_mperf_after;
> > +             uint64_t guest_aperf, guest_mperf;
> > +             struct ucall uc;
> > +
> > +             vcpu_run(vcpu);
> > +             TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
> > +
> > +             switch (get_ucall(vcpu, &uc)) {
> > +             case UCALL_DONE:
> > +                     break;
> > +             case UCALL_ABORT:
> > +                     REPORT_GUEST_ASSERT(uc);
> > +             case UCALL_SYNC:
> > +                     guest_aperf = uc.args[0];
> > +                     guest_mperf = uc.args[1];
> > +
> > +                     host_aperf_after = read_dev_msr(msr_fd, MSR_IA32_APERF);
> > +                     host_mperf_after = read_dev_msr(msr_fd, MSR_IA32_MPERF);
> > +
> > +                     TEST_ASSERT(host_aperf_before < guest_aperf,
> > +                                 "APERF: host_before (%lu) >= guest (%lu)",
> > +                                 host_aperf_before, guest_aperf);
>
> Honest question, is decimal really better than hex for these?

They are just numbers, so any base should be fine. I guess it depends
on which base you're most comfortable with. I could add a command-line
parameter.

> > +                     TEST_ASSERT(guest_aperf < host_aperf_after,
> > +                                 "APERF: guest (%lu) >= host_after (%lu)",
> > +                                 guest_aperf, host_aperf_after);
> > +                     TEST_ASSERT(host_mperf_before < guest_mperf,
> > +                                 "MPERF: host_before (%lu) >= guest (%lu)",
> > +                                 host_mperf_before, guest_mperf);
> > +                     TEST_ASSERT(guest_mperf < host_mperf_after,
> > +                                 "MPERF: guest (%lu) >= host_after (%lu)",
> > +                                 guest_mperf, host_mperf_after);
> > +
> > +                     host_aperf_before = host_aperf_after;
> > +                     host_mperf_before = host_mperf_after;
> > +
> > +                     break;
> > +             }
> > +     }
> > +
> > +     TEST_ASSERT_EQ(i, NUM_ITERATIONS);
>
> Why?

I think this was leftover from a version where it was possible to
break out of the loop early. I'll get rid of it.

V4 this week, I hope.
Re: [PATCH v3 2/2] KVM: selftests: Test behavior of KVM_X86_DISABLE_EXITS_APERFMPERF
Posted by Sean Christopherson 9 months, 1 week ago
On Wed, May 07, 2025, Jim Mattson wrote:
> On Mon, Apr 28, 2025 at 6:26 PM Sean Christopherson <seanjc@google.com> wrote:
> > > +     /*
> > > +      * This test requires a non-standard VM initialization, because
> > > +      * KVM_ENABLE_CAP cannot be used on a VM file descriptor after
> > > +      * a VCPU has been created.
> >
> > Hrm, we should really sort this out.  Every test that needs to enable a capability
> > is having to copy+paste this pattern.  I don't love the idea of expanding
> > __vm_create_with_one_vcpu(), but there's gotta be a solution that isn't horrible,
> > and anything is better than endly copy paste.
> 
> This is all your fault, I believe. But, I'll see what I can do.

Ha, that it is, both on the KVM and the selftests side.

Unless you already have something clever in hand, just keep what you have.  I poked
at this a bit today, and came to the conclusion that trying to save two lives of
"manual" effort isn't worth the explosion in APIs and complexity.  I was thinking
that the only additional input would be the capability to enable, but most usage
also needs to specify a payload, and this pattern is used in a few places where
a selftest does more than toggle a capability.

What I really want is the ability to provide a closure to all of the "create with
vCPUs" APIs, e.g.


	vm = vm_create_with_one_vcpu(&vcpu, guest_code, magic() {
		vm_enable_cap(vm, KVM_CAP_X86_DISABLE_EXITS,
			      KVM_X86_DISABLE_EXITS_APERFMPERF);
	});

But even if we managed to make something work, I'm not sure it'd be worth the
plumbing.

One thing that would make me less annoyed would be to eliminate the @vcpu_id
param, e.g.

  static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, void *guest_code)
  {
	return __vm_vcpu_add(vm, vm->nr_vcpus++, guest_code);
  }

so that at least this pattern doesn't have '0' hardcoded everywhere.  But that's
an annoying cleanup due to __vm_vcpu_add() not being a strict superset of
vm_vcpu_add(), i.e. would require a lot of churn.

So for this series, just keep the copy+pasted pattern.

> > > +      */
> > > +     vm = vm_create(1);
> > > +
> > > +     TEST_REQUIRE(kvm_can_disable_aperfmperf_exits(vm));
> >
> >         TEST_REQUIRE(vm_check_cap(vm, KVM_CAP_X86_DISABLE_EXITS) &
> >                      KVM_X86_DISABLE_EXITS_APERFMPERF);
> > > +
> > > +     vm_enable_cap(vm, KVM_CAP_X86_DISABLE_EXITS,
> > > +                   KVM_X86_DISABLE_EXITS_APERFMPERF);
> > > +
> > > +     vcpu = vm_vcpu_add(vm, 0, guest_code);
> > > +
> > > +     host_aperf_before = read_dev_msr(msr_fd, MSR_IA32_APERF);
> > > +     host_mperf_before = read_dev_msr(msr_fd, MSR_IA32_MPERF);
> > > +
> > > +     for (i = 0; i < NUM_ITERATIONS; i++) {
> > > +             uint64_t host_aperf_after, host_mperf_after;
> > > +             uint64_t guest_aperf, guest_mperf;
> > > +             struct ucall uc;
> > > +
> > > +             vcpu_run(vcpu);
> > > +             TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
> > > +
> > > +             switch (get_ucall(vcpu, &uc)) {
> > > +             case UCALL_DONE:
> > > +                     break;
> > > +             case UCALL_ABORT:
> > > +                     REPORT_GUEST_ASSERT(uc);
> > > +             case UCALL_SYNC:
> > > +                     guest_aperf = uc.args[0];
> > > +                     guest_mperf = uc.args[1];
> > > +
> > > +                     host_aperf_after = read_dev_msr(msr_fd, MSR_IA32_APERF);
> > > +                     host_mperf_after = read_dev_msr(msr_fd, MSR_IA32_MPERF);
> > > +
> > > +                     TEST_ASSERT(host_aperf_before < guest_aperf,
> > > +                                 "APERF: host_before (%lu) >= guest (%lu)",
> > > +                                 host_aperf_before, guest_aperf);
> >
> > Honest question, is decimal really better than hex for these?
> 
> They are just numbers, so any base should be fine. I guess it depends
> on which base you're most comfortable with. I could add a command-line
> parameter.

Nah, don't bother, pick whatever you like.  I was genuinely curious if one format
or another made it easier to understand the output.