From: David Woodhouse <dwmw@amazon.co.uk>

This detects the Bhyve hypervisor and enables 15-bit MSI support if
available.

Detecting Bhyve used to be a purely cosmetic issue, of the kernel printing
'Hypervisor detected: Bhyve' at boot time.

But FreeBSD 15.0 will support¹ the 15-bit MSI enlightenment to support
more than 255 vCPUs (http://david.woodhou.se/ExtDestId.pdf) which means
there's now actually some functional reason to do so.

¹ https://github.com/freebsd/freebsd-src/commit/313a68ea20b4

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 arch/x86/Kconfig                  |  9 +++++
 arch/x86/include/asm/hypervisor.h |  2 +
 arch/x86/kernel/cpu/Makefile      |  1 +
 arch/x86/kernel/cpu/bhyve.c       | 62 +++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/hypervisor.c  |  3 ++
 5 files changed, 77 insertions(+)
 create mode 100644 arch/x86/kernel/cpu/bhyve.c

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 58d890fe2100..ac1c6df44212 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -879,6 +879,15 @@ config ACRN_GUEST
 	  IOT with small footprint and real-time features. More details can be
 	  found in https://projectacrn.org/.
 
+config BHYVE_GUEST
+	bool "Bhyve (BSD Hypervisor) Guest support"
+	depends on X86_64
+	help
+	  This option allows to run Linux to recognise when it is running as a
+	  guest in the Bhyve hypervisor, and to support more than 255 vCPUs when
+	  when doing so. More details about Bhyve can be found at https://bhyve.org
+	  and https://wiki.freebsd.org/bhyve/.
+
 config INTEL_TDX_GUEST
 	bool "Intel TDX (Trust Domain Extensions) - Guest Support"
 	depends on X86_64 && CPU_SUP_INTEL
diff --git a/arch/x86/include/asm/hypervisor.h b/arch/x86/include/asm/hypervisor.h
index e41cbf2ec41d..9ad86a7d13f6 100644
--- a/arch/x86/include/asm/hypervisor.h
+++ b/arch/x86/include/asm/hypervisor.h
@@ -30,6 +30,7 @@ enum x86_hypervisor_type {
 	X86_HYPER_KVM,
 	X86_HYPER_JAILHOUSE,
 	X86_HYPER_ACRN,
+	X86_HYPER_BHYVE,
 };
 
 #ifdef CONFIG_HYPERVISOR_GUEST
@@ -64,6 +65,7 @@ extern const struct hypervisor_x86 x86_hyper_xen_pv;
 extern const struct hypervisor_x86 x86_hyper_kvm;
 extern const struct hypervisor_x86 x86_hyper_jailhouse;
 extern const struct hypervisor_x86 x86_hyper_acrn;
+extern const struct hypervisor_x86 x86_hyper_bhyve;
 extern struct hypervisor_x86 x86_hyper_xen_hvm;
 
 extern bool nopv;
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index 1e26179ff18c..2f8a58ef690e 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -58,6 +58,7 @@ obj-$(CONFIG_X86_SGX)			+= sgx/
 obj-$(CONFIG_X86_LOCAL_APIC)		+= perfctr-watchdog.o
 
 obj-$(CONFIG_HYPERVISOR_GUEST)		+= vmware.o hypervisor.o mshyperv.o
+obj-$(CONFIG_BHYVE_GUEST)		+= bhyve.o
 obj-$(CONFIG_ACRN_GUEST)		+= acrn.o
 
 obj-$(CONFIG_DEBUG_FS)			+= debugfs.o
diff --git a/arch/x86/kernel/cpu/bhyve.c b/arch/x86/kernel/cpu/bhyve.c
new file mode 100644
index 000000000000..45bb5c44788a
--- /dev/null
+++ b/arch/x86/kernel/cpu/bhyve.c
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * FreeBSD Bhyve guest enlightenments
+ *
+ * Copyright © 2025 Amazon.com, Inc. or its affiliates.
+ *
+ * Author: David Woodhouse <dwmw2@infradead.org>
+ */
+
+#include <linux/init.h>
+#include <linux/export.h>
+#include <asm/processor.h>
+#include <asm/hypervisor.h>
+
+static uint32_t bhyve_cpuid_base;
+static uint32_t bhyve_cpuid_max;
+
+#define CPUID_BHYVE_FEATURES		1
+
+/* Features advertised in CPUID_BHYVE_FEATURES %eax */
+#define CPUID_BHYVE_FEAT_EXT_DEST_ID	(1UL << 0) /* MSI Extended Dest ID */
+
+static uint32_t __init bhyve_detect(void)
+{
+	if (boot_cpu_data.cpuid_level < 0 ||
+            !cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
+                return 0;
+
+	bhyve_cpuid_base = cpuid_base_hypervisor("bhyve bhyve ", 0);
+	if (!bhyve_cpuid_base)
+		return 0;
+
+	bhyve_cpuid_max = cpuid_eax(bhyve_cpuid_max);
+	return bhyve_cpuid_max;
+}
+
+static uint32_t bhyve_features(void)
+{
+	if (bhyve_cpuid_max < bhyve_cpuid_base + CPUID_BHYVE_FEATURES)
+		return 0;
+
+	return cpuid_eax(bhyve_cpuid_base + CPUID_BHYVE_FEATURES);
+}
+
+static bool __init bhyve_ext_dest_id(void)
+{
+	return !!(bhyve_features() & CPUID_BHYVE_FEAT_EXT_DEST_ID);
+}
+
+static bool __init bhyve_x2apic_available(void)
+{
+	/* Bhyve has always supported x2apic */
+	return true;
+}
+
+const struct hypervisor_x86 x86_hyper_bhyve __refconst = {
+	.name			= "Bhyve",
+	.detect			= bhyve_detect,
+	.init.init_platform	= x86_init_noop,
+	.init.x2apic_available	= bhyve_x2apic_available,
+	.init.msi_ext_dest_id	= bhyve_ext_dest_id,
+};
diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
index 553bfbfc3a1b..f3e9219845e8 100644
--- a/arch/x86/kernel/cpu/hypervisor.c
+++ b/arch/x86/kernel/cpu/hypervisor.c
@@ -45,6 +45,9 @@ static const __initconst struct hypervisor_x86 * const hypervisors[] =
 #ifdef CONFIG_ACRN_GUEST
 	&x86_hyper_acrn,
 #endif
+#ifdef CONFIG_BHYVE_GUEST
+	&x86_hyper_bhyve,
+#endif
 };
 
 enum x86_hypervisor_type x86_hyper_type;
-- 
2.49.0

On Thu, 14 Aug 2025, David Woodhouse wrote:
> +
> +static uint32_t bhyve_cpuid_base;
> +static uint32_t bhyve_cpuid_max;
> +
...
> +static uint32_t __init bhyve_detect(void)
> +{
> +	if (boot_cpu_data.cpuid_level < 0 ||
> +            !cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
> +                return 0;
> +
> +	bhyve_cpuid_base = cpuid_base_hypervisor("bhyve bhyve ", 0);
> +	if (!bhyve_cpuid_base)
> +		return 0;
> +
> +	bhyve_cpuid_max = cpuid_eax(bhyve_cpuid_max);
> +
                                                ^^^

You're using CPUID(0x0) by mistake here ;)

I guess you just meant:

	bhyve_cpuid_max = cpuid_eax(bhyve_cpuid_base);

Thanks!
Ahmed

On Thu, 14 Aug 2025, David Woodhouse wrote:
>
> +static uint32_t __init bhyve_detect(void)
> +{
> +	if (boot_cpu_data.cpuid_level < 0 ||
> ...

The CPUID API at <asm/cpuid/api.h> provides a cpuid_feature() macro for
this.  Let's please use that instead.

I understand that arch/x86/kernel/kvm.c and arch/x86/kernel/jailhouse.c
does a similar "cpuid_level < 0" check, but they should also be using
cpuid_feature() instead.

Thanks!
Ahmed

On Thu, 2025-08-14 at 19:10 +0200, Ahmed S. Darwish wrote:
> On Thu, 14 Aug 2025, David Woodhouse wrote:
> > 
> > +static uint32_t __init bhyve_detect(void)
> > +{
> > +	if (boot_cpu_data.cpuid_level < 0 ||
> > ...
> 
> The CPUID API at <asm/cpuid/api.h> provides a cpuid_feature() macro for
> this.  Let's please use that instead.


> I understand that arch/x86/kernel/kvm.c and arch/x86/kernel/jailhouse.c
> does a similar "cpuid_level < 0" check, but they should also be using
> cpuid_feature() instead.


Or just not, in the case of jailhouse and bhyve? Since cpuid_feature()
is hard-coded to true for x86_64 anyway, and they both depend on that
(like acrn, which already only checks for X86_FEATURE_HYPERVISOR).

And anyway, how is X86_FEATURE_HYPERVISOR even going to get set if
there's no CPUID? Can that happen?


Speaking of which, I note detect_hypervisor_vendor does a bunch of
calling into various hypervisor detection routines which are only going
to return immediately because !X86_FEATURE_HYPERVISOR. Should that be
lifted out to a flag like the ignore_nopv flag, so it doesn't need to
bother, and the detection routines don't all need to reimplement the
check differently for themselves?

On Fri, 15 Aug 2025, David Woodhouse wrote:
>
> Or just not, in the case of jailhouse and bhyve? Since cpuid_feature()
> is hard-coded to true for x86_64 anyway, and they both depend on that
> (like acrn, which already only checks for X86_FEATURE_HYPERVISOR).
>
> And anyway, how is X86_FEATURE_HYPERVISOR even going to get set if
> there's no CPUID? Can that happen?
>

Correct, these early-boot feature detection code paths are all already
messed up.  I will clean them as part of the CPUID parser work.

For example, in the CPUID parser patch queue next iteration (v5),
X86_FEATURE flags are integrated.  There's also a Directed Acylic Graph
of X86_FEATURE and CPUID bits dependencies.

So, in the above case, the dependencies table will just have:

    X86_FEATURE_HYPERVISOR ► X86_FEATURE_CPUID (and whatever else)

and all the circles of x86 vendor checking, CPUID max level checking,
other CPUID-bits checking, X86_FEATURE dependency checking, etc. will all
be removed from call sites.

I will show more of that at LPC Japan x86 track, and hopefully much
before that the CPUID parser v5 shall also be already posted to LKML.

Thanks!
Ahmed

On Thu, 14 Aug 2025, David Woodhouse wrote:
> +
> +static uint32_t __init bhyve_detect(void)
> +{
> +	if (boot_cpu_data.cpuid_level < 0 ||
> +            !cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
> +                return 0;
> +
> +	bhyve_cpuid_base = cpuid_base_hypervisor("bhyve bhyve ", 0);

KVM has KVM_SIGNATURE, and Xen has XEN_SIGNATURE.

So, can we please assign a BHYVE_SIGNATURE symbol to this?

It will just make my job easier when I'm converting all this later :)

Thanks,
Ahmed