[v1] x86/bhyve: Detect FreeBSD Bhyve hypervisor

[RFC PATCH] x86/bhyve: Detect FreeBSD Bhyve hypervisor

Posted by David Woodhouse 6 months ago

From: David Woodhouse <dwmw@amazon.co.uk>

This detects the Bhyve hypervisor and enables 15-bit MSI support if
available.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
RFC because the CPUID_BHYVE_FEATURES leaf is submitted to FreeBSD in
https://github.com/freebsd/freebsd-src/pull/1797 and not yet merged.
And because I haven't tested this patch at all other than building it.

It's tiny so I didn't put it behind a separate CONFIG_BHYVE_GUEST.
Should I?

 arch/x86/include/asm/hypervisor.h |  2 +
 arch/x86/kernel/cpu/Makefile      |  2 +-
 arch/x86/kernel/cpu/bhyve.c       | 62 +++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/hypervisor.c  |  1 +
 4 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 arch/x86/kernel/cpu/bhyve.c

diff --git a/arch/x86/include/asm/hypervisor.h b/arch/x86/include/asm/hypervisor.h
index e41cbf2ec41d..9ad86a7d13f6 100644
--- a/arch/x86/include/asm/hypervisor.h
+++ b/arch/x86/include/asm/hypervisor.h
@@ -30,6 +30,7 @@ enum x86_hypervisor_type {
 	X86_HYPER_KVM,
 	X86_HYPER_JAILHOUSE,
 	X86_HYPER_ACRN,
+	X86_HYPER_BHYVE,
 };
 
 #ifdef CONFIG_HYPERVISOR_GUEST
@@ -64,6 +65,7 @@ extern const struct hypervisor_x86 x86_hyper_xen_pv;
 extern const struct hypervisor_x86 x86_hyper_kvm;
 extern const struct hypervisor_x86 x86_hyper_jailhouse;
 extern const struct hypervisor_x86 x86_hyper_acrn;
+extern const struct hypervisor_x86 x86_hyper_bhyve;
 extern struct hypervisor_x86 x86_hyper_xen_hvm;
 
 extern bool nopv;
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index 1e26179ff18c..d4cb04edbacf 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -57,7 +57,7 @@ obj-$(CONFIG_X86_SGX)			+= sgx/
 
 obj-$(CONFIG_X86_LOCAL_APIC)		+= perfctr-watchdog.o
 
-obj-$(CONFIG_HYPERVISOR_GUEST)		+= vmware.o hypervisor.o mshyperv.o
+obj-$(CONFIG_HYPERVISOR_GUEST)		+= vmware.o hypervisor.o mshyperv.o bhyve.o
 obj-$(CONFIG_ACRN_GUEST)		+= acrn.o
 
 obj-$(CONFIG_DEBUG_FS)			+= debugfs.o
diff --git a/arch/x86/kernel/cpu/bhyve.c b/arch/x86/kernel/cpu/bhyve.c
new file mode 100644
index 000000000000..6d8d6ba458ef
--- /dev/null
+++ b/arch/x86/kernel/cpu/bhyve.c
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * FreeBSD Bhyve guest enlightenments
+ *
+ * Copyright © 2025 Amazon.com, Inc. or its affiliates.
+ *
+ * Author: David Woodhouse <dwmw2@infradead.org>
+ */
+
+#include <linux/init.h>
+#include <linux/export.h>
+#include <asm/processor.h>
+#include <asm/hypervisor.h>
+
+static uint32_t bhyve_cpuid_base;
+static uint32_t bhyve_cpuid_max;
+
+#define CPUID_BHYVE_FEATURES		1
+
+/* Features advertised in CPUID_BHYVE_FEATURES %eax */
+#define CPUID_BHYVE_FEAT_EXT_DEST_ID	(1UL << 0) /* MSI Extended Dest ID */
+
+static uint32_t __init bhyve_detect(void)
+{
+	if (boot_cpu_data.cpuid_level < 0 ||
+            !boot_cpu_has(X86_FEATURE_HYPERVISOR))
+                return 0;
+
+	bhyve_cpuid_base = cpuid_base_hypervisor("bhyve bhyve ", 0);
+	if (!bhyve_cpuid_base)
+		return 0;
+
+	bhyve_cpuid_max = cpuid_eax(bhyve_cpuid_max);
+	return bhyve_cpuid_max;
+}
+
+static uint32_t bhyve_features(void)
+{
+	if (bhyve_cpuid_max < bhyve_cpuid_base + CPUID_BHYVE_FEATURES)
+		return 0;
+
+	return cpuid_eax(bhyve_cpuid_base + CPUID_BHYVE_FEATURES);
+}
+
+static bool __init bhyve_ext_dest_id(void)
+{
+	return !!(bhyve_features() & CPUID_BHYVE_FEAT_EXT_DEST_ID);
+}
+
+static bool __init bhyve_x2apic_available(void)
+{
+	/* Bhyve has always supported x2apic */
+	return true;
+}
+
+const struct hypervisor_x86 x86_hyper_bhyve __refconst = {
+	.name			= "Bhyve",
+	.detect			= bhyve_detect,
+	.init.init_platform	= x86_init_noop,
+	.init.x2apic_available	= bhyve_x2apic_available,
+	.init.msi_ext_dest_id	= bhyve_ext_dest_id,
+};
diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
index 553bfbfc3a1b..157cf9edeb35 100644
--- a/arch/x86/kernel/cpu/hypervisor.c
+++ b/arch/x86/kernel/cpu/hypervisor.c
@@ -36,6 +36,7 @@ static const __initconst struct hypervisor_x86 * const hypervisors[] =
 #endif
 	&x86_hyper_vmware,
 	&x86_hyper_ms_hyperv,
+	&x86_hyper_bhyve,
 #ifdef CONFIG_KVM_GUEST
 	&x86_hyper_kvm,
 #endif
-- 
2.49.0

Re: [RFC PATCH] x86/bhyve: Detect FreeBSD Bhyve hypervisor

Posted by Borislav Petkov 5 months, 3 weeks ago

On Mon, Aug 11, 2025 at 01:16:42PM +0200, David Woodhouse wrote:
> From: David Woodhouse <dwmw@amazon.co.uk>
> 
> This detects the Bhyve hypervisor

Users of this are?

Wikipedia says it already supports Linux :-P

> and enables 15-bit MSI support if
> available.
> 
> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> ---
> RFC because the CPUID_BHYVE_FEATURES leaf is submitted to FreeBSD in

Which makes it, what, 0x4000_0001?

I presume there's an agreement among all the hypervisors...?

> https://github.com/freebsd/freebsd-src/pull/1797 and not yet merged.
> And because I haven't tested this patch at all other than building it.
> 
> It's tiny so I didn't put it behind a separate CONFIG_BHYVE_GUEST.
> Should I?

Yeah, please do.

> +/* Features advertised in CPUID_BHYVE_FEATURES %eax */
> +#define CPUID_BHYVE_FEAT_EXT_DEST_ID	(1UL << 0) /* MSI Extended Dest ID */
> +
> +static uint32_t __init bhyve_detect(void)
> +{
> +	if (boot_cpu_data.cpuid_level < 0 ||
> +            !boot_cpu_has(X86_FEATURE_HYPERVISOR))

cpu_feature_enabled()

-- 
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette

Re: [RFC PATCH] x86/bhyve: Detect FreeBSD Bhyve hypervisor

Posted by David Woodhouse 5 months, 3 weeks ago

On Thu, 2025-08-14 at 12:15 +0200, Borislav Petkov wrote:
> On Mon, Aug 11, 2025 at 01:16:42PM +0200, David Woodhouse wrote:
> > From: David Woodhouse <dwmw@amazon.co.uk>
> > 
> > This detects the Bhyve hypervisor
> 
> Users of this are?

FreeBSD and Illumos (and some other *BSD) users, presumably?

> Wikipedia says it already supports Linux :-P

It does, but Linux doesn't support *it*. Which mostly didn't matter
because all that's missing was a cosmetic 'Hypervisor detected: Bhyve'
in the kernel log. But now it's no longer just cosmetic because...

> > and enables 15-bit MSI support if available.

I have finally written up some proper documentation for that 15-bit MSI
enlightenment, btw: http://david.woodhou.se/ExtDestId.pdf

> > Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> > ---
> > RFC because the CPUID_BHYVE_FEATURES leaf is submitted to FreeBSD in
> 
> Which makes it, what, 0x4000_0001?
> 
> I presume there's an agreement among all the hypervisors...?

There is no such agreement about CPUID; by *coincidence* most of them
seem to use 0x4000_0?01 as their feature leaf, but the feature bits in
each such leaf are completely different and the guest doesn't look at a
given leaf unless it knows that specific hypervisor. Which is why the
15-bit enlightenment is advertised differently by each hypervisor
despite being precisely the same enlightenment across them all.

(Digressing a little... the slight exception is the 0x4000_0010 leaf
which by coincidence I have *also* been looking at this week. VMware
proposed a common convention in https://lkml.org/lkml/2008/10/1/246 but
it was shot down in flames. However, the 'timing info' leaf didn't
quite die, and both XNU and FreeBSD guests actually look for it at
0x4000_0010 under *all* hypervisors. And it even got added in the EC2
Nitro hypervisor. I have patches pending to make KVM support it on both
host and guest side, which I'll post shortly.)

> > https://github.com/freebsd/freebsd-src/pull/1797 and not yet merged.
> > And because I haven't tested this patch at all other than building it.
> > 
> > It's tiny so I didn't put it behind a separate CONFIG_BHYVE_GUEST.
> > Should I?
> 
> Yeah, please do.
> 
> > +/* Features advertised in CPUID_BHYVE_FEATURES %eax */
> > +#define CPUID_BHYVE_FEAT_EXT_DEST_ID	(1UL << 0) /* MSI Extended Dest ID */
> > +
> > +static uint32_t __init bhyve_detect(void)
> > +{
> > +	if (boot_cpu_data.cpuid_level < 0 ||
> > +            !boot_cpu_has(X86_FEATURE_HYPERVISOR))
> 
> cpu_feature_enabled()
> 

Ack, v2 coming up shortly. Thanks.

Re: [RFC PATCH] x86/bhyve: Detect FreeBSD Bhyve hypervisor

Posted by Ahmed S. Darwish 5 months, 3 weeks ago

On Thu, 14 Aug 2025, David Woodhouse wrote:
> On Thu, 2025-08-14 at 12:15 +0200, Borislav Petkov wrote:
>
> >
> > Which makes it, what, 0x4000_0001?
> >
> > I presume there's an agreement among all the hypervisors...?
>
> There is no such agreement about CPUID; by *coincidence* most of them
> seem to use 0x4000_0?01 as their feature leaf, but the feature bits in
> each such leaf are completely different and the guest doesn't look at a
> given leaf unless it knows that specific hypervisor. Which is why the
> 15-bit enlightenment is advertised differently by each hypervisor
> despite being precisely the same enlightenment across them all.
>

For what it takes, such hypervisor leaves, in the 0x40000000 range, will
be converted by the ongoing x86-cpuid-db CPUID parser work at some point.

(Another iteration of the CPUID parser is to be posted today.)

Thanks,
Ahmed

Re: [RFC PATCH] x86/bhyve: Detect FreeBSD Bhyve hypervisor

Posted by David Woodhouse 5 months, 4 weeks ago

On Mon, 2025-08-11 at 13:16 +0200, David Woodhouse wrote:
> From: David Woodhouse <dwmw@amazon.co.uk>
> 
> This detects the Bhyve hypervisor and enables 15-bit MSI support if
> available.
> 
> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> ---
> RFC because the CPUID_BHYVE_FEATURES leaf is submitted to FreeBSD in
> https://github.com/freebsd/freebsd-src/pull/1797 and not yet merged.
> And because I haven't tested this patch at all other than building it.

Now merged and should be in FreeBSD 15.0 release. Thanks, Colin!
https://github.com/freebsd/freebsd-src/commit/4322d597453d6

> It's tiny so I didn't put it behind a separate CONFIG_BHYVE_GUEST.
> Should I?
> 
>  arch/x86/include/asm/hypervisor.h |  2 +
>  arch/x86/kernel/cpu/Makefile      |  2 +-
>  arch/x86/kernel/cpu/bhyve.c       | 62 +++++++++++++++++++++++++++++++
>  arch/x86/kernel/cpu/hypervisor.c  |  1 +
>  4 files changed, 66 insertions(+), 1 deletion(-)
>  create mode 100644 arch/x86/kernel/cpu/bhyve.c
> 
> diff --git a/arch/x86/include/asm/hypervisor.h b/arch/x86/include/asm/hypervisor.h
> index e41cbf2ec41d..9ad86a7d13f6 100644
> --- a/arch/x86/include/asm/hypervisor.h
> +++ b/arch/x86/include/asm/hypervisor.h
> @@ -30,6 +30,7 @@ enum x86_hypervisor_type {
>  	X86_HYPER_KVM,
>  	X86_HYPER_JAILHOUSE,
>  	X86_HYPER_ACRN,
> +	X86_HYPER_BHYVE,
>  };
>  
>  #ifdef CONFIG_HYPERVISOR_GUEST
> @@ -64,6 +65,7 @@ extern const struct hypervisor_x86 x86_hyper_xen_pv;
>  extern const struct hypervisor_x86 x86_hyper_kvm;
>  extern const struct hypervisor_x86 x86_hyper_jailhouse;
>  extern const struct hypervisor_x86 x86_hyper_acrn;
> +extern const struct hypervisor_x86 x86_hyper_bhyve;
>  extern struct hypervisor_x86 x86_hyper_xen_hvm;
>  
>  extern bool nopv;
> diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
> index 1e26179ff18c..d4cb04edbacf 100644
> --- a/arch/x86/kernel/cpu/Makefile
> +++ b/arch/x86/kernel/cpu/Makefile
> @@ -57,7 +57,7 @@ obj-$(CONFIG_X86_SGX)			+= sgx/
>  
>  obj-$(CONFIG_X86_LOCAL_APIC)		+= perfctr-watchdog.o
>  
> -obj-$(CONFIG_HYPERVISOR_GUEST)		+= vmware.o hypervisor.o mshyperv.o
> +obj-$(CONFIG_HYPERVISOR_GUEST)		+= vmware.o hypervisor.o mshyperv.o bhyve.o
>  obj-$(CONFIG_ACRN_GUEST)		+= acrn.o
>  
>  obj-$(CONFIG_DEBUG_FS)			+= debugfs.o
> diff --git a/arch/x86/kernel/cpu/bhyve.c b/arch/x86/kernel/cpu/bhyve.c
> new file mode 100644
> index 000000000000..6d8d6ba458ef
> --- /dev/null
> +++ b/arch/x86/kernel/cpu/bhyve.c
> @@ -0,0 +1,62 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * FreeBSD Bhyve guest enlightenments
> + *
> + * Copyright © 2025 Amazon.com, Inc. or its affiliates.
> + *
> + * Author: David Woodhouse <dwmw2@infradead.org>
> + */
> +
> +#include <linux/init.h>
> +#include <linux/export.h>
> +#include <asm/processor.h>
> +#include <asm/hypervisor.h>
> +
> +static uint32_t bhyve_cpuid_base;
> +static uint32_t bhyve_cpuid_max;
> +
> +#define CPUID_BHYVE_FEATURES		1
> +
> +/* Features advertised in CPUID_BHYVE_FEATURES %eax */
> +#define CPUID_BHYVE_FEAT_EXT_DEST_ID	(1UL << 0) /* MSI Extended Dest ID */
> +
> +static uint32_t __init bhyve_detect(void)
> +{
> +	if (boot_cpu_data.cpuid_level < 0 ||
> +            !boot_cpu_has(X86_FEATURE_HYPERVISOR))
> +                return 0;
> +
> +	bhyve_cpuid_base = cpuid_base_hypervisor("bhyve bhyve ", 0);
> +	if (!bhyve_cpuid_base)
> +		return 0;
> +
> +	bhyve_cpuid_max = cpuid_eax(bhyve_cpuid_max);
> +	return bhyve_cpuid_max;
> +}
> +
> +static uint32_t bhyve_features(void)
> +{
> +	if (bhyve_cpuid_max < bhyve_cpuid_base + CPUID_BHYVE_FEATURES)
> +		return 0;
> +
> +	return cpuid_eax(bhyve_cpuid_base + CPUID_BHYVE_FEATURES);
> +}
> +
> +static bool __init bhyve_ext_dest_id(void)
> +{
> +	return !!(bhyve_features() & CPUID_BHYVE_FEAT_EXT_DEST_ID);
> +}
> +
> +static bool __init bhyve_x2apic_available(void)
> +{
> +	/* Bhyve has always supported x2apic */
> +	return true;
> +}
> +
> +const struct hypervisor_x86 x86_hyper_bhyve __refconst = {
> +	.name			= "Bhyve",
> +	.detect			= bhyve_detect,
> +	.init.init_platform	= x86_init_noop,
> +	.init.x2apic_available	= bhyve_x2apic_available,
> +	.init.msi_ext_dest_id	= bhyve_ext_dest_id,
> +};
> diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
> index 553bfbfc3a1b..157cf9edeb35 100644
> --- a/arch/x86/kernel/cpu/hypervisor.c
> +++ b/arch/x86/kernel/cpu/hypervisor.c
> @@ -36,6 +36,7 @@ static const __initconst struct hypervisor_x86 * const hypervisors[] =
>  #endif
>  	&x86_hyper_vmware,
>  	&x86_hyper_ms_hyperv,
> +	&x86_hyper_bhyve,
>  #ifdef CONFIG_KVM_GUEST
>  	&x86_hyper_kvm,
>  #endif