IBPB mitigation for VMSCAPE is an overkill for CPUs that are only affected
by the BHI variant of VMSCAPE. On such CPUs, eIBRS already provides
indirect branch isolation between guest and host userspace. But, a guest
could still poison the branch history.
To mitigate that, use the recently added clear_bhb_long_loop() to isolate
the branch history between guest and userspace. Add cmdline option
'vmscape=on' that automatically selects the appropriate mitigation based
on the CPU.
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
Documentation/admin-guide/hw-vuln/vmscape.rst | 8 ++++
Documentation/admin-guide/kernel-parameters.txt | 4 +-
arch/x86/include/asm/cpufeatures.h | 1 +
arch/x86/include/asm/entry-common.h | 12 +++---
arch/x86/include/asm/nospec-branch.h | 2 +-
arch/x86/kernel/cpu/bugs.c | 53 ++++++++++++++++++-------
arch/x86/kvm/x86.c | 5 ++-
7 files changed, 61 insertions(+), 24 deletions(-)
diff --git a/Documentation/admin-guide/hw-vuln/vmscape.rst b/Documentation/admin-guide/hw-vuln/vmscape.rst
index d9b9a2b6c114c05a7325e5f3c9d42129339b870b..580f288ae8bfc601ff000d6d95d711bb9084459e 100644
--- a/Documentation/admin-guide/hw-vuln/vmscape.rst
+++ b/Documentation/admin-guide/hw-vuln/vmscape.rst
@@ -86,6 +86,10 @@ The possible values in this file are:
run a potentially malicious guest and issues an IBPB before the first
exit to userspace after VM-exit.
+ * 'Mitigation: Clear BHB before exit to userspace':
+
+ As above, conditional BHB clearing mitigation is enabled.
+
* 'Mitigation: IBPB on VMEXIT':
IBPB is issued on every VM-exit. This occurs when other mitigations like
@@ -108,3 +112,7 @@ The mitigation can be controlled via the ``vmscape=`` command line parameter:
Force vulnerability detection and mitigation even on processors that are
not known to be affected.
+
+ * ``vmscape=on``:
+
+ Choose the mitigation based on the VMSCAPE variant the CPU is affected by.
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 6c42061ca20e581b5192b66c6f25aba38d4f8ff8..4b4711ced5e187495476b5365cd7b3df81db893b 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -8104,9 +8104,11 @@
off - disable the mitigation
ibpb - use Indirect Branch Prediction Barrier
- (IBPB) mitigation (default)
+ (IBPB) mitigation
force - force vulnerability detection even on
unaffected processors
+ on - (default) automatically select IBPB
+ or BHB clear mitigation based on CPU
vsyscall= [X86-64,EARLY]
Controls the behavior of vsyscalls (i.e. calls to
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 4091a776e37aaed67ca93b0a0cd23cc25dbc33d4..3d547c3eab4e3290de3eee8e89f21587fee34931 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -499,6 +499,7 @@
#define X86_FEATURE_IBPB_EXIT_TO_USER (21*32+14) /* Use IBPB on exit-to-userspace, see VMSCAPE bug */
#define X86_FEATURE_ABMC (21*32+15) /* Assignable Bandwidth Monitoring Counters */
#define X86_FEATURE_MSR_IMM (21*32+16) /* MSR immediate form instructions */
+#define X86_FEATURE_CLEAR_BHB_EXIT_TO_USER (21*32+17) /* Clear branch history on exit-to-userspace, see VMSCAPE bug */
/*
* BUG word(s)
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index ce3eb6d5fdf9f2dba59b7bad24afbfafc8c36918..b7b9af1b641385b8283edf2449578ff65e5bd6df 100644
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -94,11 +94,13 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
*/
choose_random_kstack_offset(rdtsc());
- /* Avoid unnecessary reads of 'x86_ibpb_exit_to_user' */
- if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) &&
- this_cpu_read(x86_ibpb_exit_to_user)) {
- indirect_branch_prediction_barrier();
- this_cpu_write(x86_ibpb_exit_to_user, false);
+ if (unlikely(this_cpu_read(x86_pred_flush_pending))) {
+ if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
+ indirect_branch_prediction_barrier();
+ else if (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER))
+ clear_bhb_long_loop();
+
+ this_cpu_write(x86_pred_flush_pending, false);
}
}
#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 49707e563bdf71bdd05d3827f10dd2b8ac6bca2c..00730cc22c2e7115f6dbb38a1ed8d10383ada5c0 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -534,7 +534,7 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
: "memory");
}
-DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user);
+DECLARE_PER_CPU(bool, x86_pred_flush_pending);
static inline void indirect_branch_prediction_barrier(void)
{
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 6a526ae1fe9933229947db5b7676a18328fe2204..02fd37bf4e6d77494c72806775f4415a27652206 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -109,12 +109,11 @@ DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current);
/*
- * Set when the CPU has run a potentially malicious guest. An IBPB will
- * be needed to before running userspace. That IBPB will flush the branch
- * predictor content.
+ * Set when the CPU has run a potentially malicious guest. Indicates that a
+ * branch predictor flush is needed before running userspace.
*/
-DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user);
-EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user);
+DEFINE_PER_CPU(bool, x86_pred_flush_pending);
+EXPORT_PER_CPU_SYMBOL_GPL(x86_pred_flush_pending);
u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
@@ -3202,13 +3201,15 @@ enum vmscape_mitigations {
VMSCAPE_MITIGATION_AUTO,
VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER,
VMSCAPE_MITIGATION_IBPB_ON_VMEXIT,
+ VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER,
};
static const char * const vmscape_strings[] = {
- [VMSCAPE_MITIGATION_NONE] = "Vulnerable",
+ [VMSCAPE_MITIGATION_NONE] = "Vulnerable",
/* [VMSCAPE_MITIGATION_AUTO] */
- [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER] = "Mitigation: IBPB before exit to userspace",
- [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT",
+ [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER] = "Mitigation: IBPB before exit to userspace",
+ [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT",
+ [VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER] = "Mitigation: Clear BHB before exit to userspace",
};
static enum vmscape_mitigations vmscape_mitigation __ro_after_init =
@@ -3226,6 +3227,8 @@ static int __init vmscape_parse_cmdline(char *str)
} else if (!strcmp(str, "force")) {
setup_force_cpu_bug(X86_BUG_VMSCAPE);
vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
+ } else if (!strcmp(str, "on")) {
+ vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
} else {
pr_err("Ignoring unknown vmscape=%s option.\n", str);
}
@@ -3236,18 +3239,35 @@ early_param("vmscape", vmscape_parse_cmdline);
static void __init vmscape_select_mitigation(void)
{
- if (!boot_cpu_has_bug(X86_BUG_VMSCAPE) ||
- !boot_cpu_has(X86_FEATURE_IBPB)) {
+ if (!boot_cpu_has_bug(X86_BUG_VMSCAPE)) {
vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
return;
}
- if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO) {
- if (should_mitigate_vuln(X86_BUG_VMSCAPE))
- vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER;
- else
- vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
+ if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO &&
+ !should_mitigate_vuln(X86_BUG_VMSCAPE))
+ vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
+
+ if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER &&
+ !boot_cpu_has(X86_FEATURE_IBPB)) {
+ pr_err("IBPB not supported, switching to AUTO select\n");
+ vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
}
+
+ if (vmscape_mitigation != VMSCAPE_MITIGATION_AUTO)
+ return;
+
+ /*
+ * CPUs with BHI_CTRL(ADL and newer) can avoid the IBPB and use BHB
+ * clear sequence. These CPUs are only vulnerable to the BHI variant
+ * of the VMSCAPE attack and does not require an IBPB flush.
+ */
+ if (boot_cpu_has(X86_FEATURE_BHI_CTRL))
+ vmscape_mitigation = VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER;
+ else if (boot_cpu_has(X86_FEATURE_IBPB))
+ vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER;
+ else
+ vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
}
static void __init vmscape_update_mitigation(void)
@@ -3266,6 +3286,8 @@ static void __init vmscape_apply_mitigation(void)
{
if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER)
setup_force_cpu_cap(X86_FEATURE_IBPB_EXIT_TO_USER);
+ else if (vmscape_mitigation == VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER)
+ setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER);
}
#undef pr_fmt
@@ -3357,6 +3379,7 @@ void cpu_bugs_smt_update(void)
break;
case VMSCAPE_MITIGATION_IBPB_ON_VMEXIT:
case VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER:
+ case VMSCAPE_MITIGATION_BHB_CLEAR_EXIT_TO_USER:
/*
* Hypervisors can be attacked across-threads, warn for SMT when
* STIBP is not already enabled system-wide.
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 42ecd093bb4c8ecfae2523b52b85779ca1e56bb5..57d26dbb43e9115880e92e448e4c018e03dca063 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -11397,8 +11397,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
* set for the CPU that actually ran the guest, and not the CPU that it
* may migrate to.
*/
- if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
- this_cpu_write(x86_ibpb_exit_to_user, true);
+ if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) ||
+ cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER))
+ this_cpu_write(x86_pred_flush_pending, true);
/*
* Consume any pending interrupts, including the possible source of
--
2.34.1On Wed, Oct 15, 2025 at 06:52:11PM -0700, Pawan Gupta wrote:
> IBPB mitigation for VMSCAPE is an overkill for CPUs that are only affected
> by the BHI variant of VMSCAPE. On such CPUs, eIBRS already provides
> indirect branch isolation between guest and host userspace. But, a guest
> could still poison the branch history.
>
> To mitigate that, use the recently added clear_bhb_long_loop() to isolate
> the branch history between guest and userspace. Add cmdline option
> 'vmscape=on' that automatically selects the appropriate mitigation based
> on the CPU.
[...]
> diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
> index ce3eb6d5fdf9f2dba59b7bad24afbfafc8c36918..b7b9af1b641385b8283edf2449578ff65e5bd6df 100644
> --- a/arch/x86/include/asm/entry-common.h
> +++ b/arch/x86/include/asm/entry-common.h
> @@ -94,11 +94,13 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
> */
> choose_random_kstack_offset(rdtsc());
>
> - /* Avoid unnecessary reads of 'x86_ibpb_exit_to_user' */
> - if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) &&
> - this_cpu_read(x86_ibpb_exit_to_user)) {
> - indirect_branch_prediction_barrier();
> - this_cpu_write(x86_ibpb_exit_to_user, false);
> + if (unlikely(this_cpu_read(x86_pred_flush_pending))) {
> + if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
> + indirect_branch_prediction_barrier();
> + else if (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_EXIT_TO_USER))
I realize that IBPB and BHB clear doesn't have to be mutually exclusive.
IBPB does avoids the need to clear BHB because it flushes the indirect
branches and BHB isn't useful anymore. But, this code doesn't need to
prevent both from being executed. This should be enforced during mitigation
selection. Updating the patch to allow both here.
> + clear_bhb_long_loop();
> +
> + this_cpu_write(x86_pred_flush_pending, false);
> }
> }
On Wed, Oct 15, 2025, Pawan Gupta wrote: > diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h > index 49707e563bdf71bdd05d3827f10dd2b8ac6bca2c..00730cc22c2e7115f6dbb38a1ed8d10383ada5c0 100644 > --- a/arch/x86/include/asm/nospec-branch.h > +++ b/arch/x86/include/asm/nospec-branch.h > @@ -534,7 +534,7 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) > : "memory"); > } > > -DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user); > +DECLARE_PER_CPU(bool, x86_pred_flush_pending); Rather than "flush pending", what about using "need" in the name to indicate that a flush is necessary? That makes it more obvious that e.g. KVM is marking the CPU as needing a flush by some other code, as opposed to implying that KVM itself has a pending flush. And maybe spell out "prediction"? Without the context of features being checked, I don't know that I would be able to guess "prediction". E.g. x86_need_prediction_flush? Or x86_prediction_flush_exit_to_user if we would prefer to clarify when the flush needs to occur?
On Mon, Oct 20, 2025 at 09:10:17AM -0700, Sean Christopherson wrote: > On Wed, Oct 15, 2025, Pawan Gupta wrote: > > diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h > > index 49707e563bdf71bdd05d3827f10dd2b8ac6bca2c..00730cc22c2e7115f6dbb38a1ed8d10383ada5c0 100644 > > --- a/arch/x86/include/asm/nospec-branch.h > > +++ b/arch/x86/include/asm/nospec-branch.h > > @@ -534,7 +534,7 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) > > : "memory"); > > } > > > > -DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user); > > +DECLARE_PER_CPU(bool, x86_pred_flush_pending); > > Rather than "flush pending", what about using "need" in the name to indicate that > a flush is necessary? That makes it more obvious that e.g. KVM is marking the > CPU as needing a flush by some other code, as opposed to implying that KVM itself > has a pending flush. > > And maybe spell out "prediction"? Without the context of features being checked, > I don't know that I would be able to guess "prediction". > > E.g. x86_need_prediction_flush? > > Or x86_prediction_flush_exit_to_user if we would prefer to clarify when the flush > needs to occur? Ok, ya this is more clear. I would want to make a small change, instead of "prediction_flush", "predictor_flush" reads better to me. Changing it to: x86_predictor_flush_exit_to_user.
On Mon, Oct 20, 2025, Pawan Gupta wrote: > On Mon, Oct 20, 2025 at 09:10:17AM -0700, Sean Christopherson wrote: > > On Wed, Oct 15, 2025, Pawan Gupta wrote: > > > diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h > > > index 49707e563bdf71bdd05d3827f10dd2b8ac6bca2c..00730cc22c2e7115f6dbb38a1ed8d10383ada5c0 100644 > > > --- a/arch/x86/include/asm/nospec-branch.h > > > +++ b/arch/x86/include/asm/nospec-branch.h > > > @@ -534,7 +534,7 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) > > > : "memory"); > > > } > > > > > > -DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user); > > > +DECLARE_PER_CPU(bool, x86_pred_flush_pending); > > > > Rather than "flush pending", what about using "need" in the name to indicate that > > a flush is necessary? That makes it more obvious that e.g. KVM is marking the > > CPU as needing a flush by some other code, as opposed to implying that KVM itself > > has a pending flush. > > > > And maybe spell out "prediction"? Without the context of features being checked, > > I don't know that I would be able to guess "prediction". > > > > E.g. x86_need_prediction_flush? > > > > Or x86_prediction_flush_exit_to_user if we would prefer to clarify when the flush > > needs to occur? > > Ok, ya this is more clear. I would want to make a small change, instead of > "prediction_flush", "predictor_flush" reads better to me. Changing it to: > x86_predictor_flush_exit_to_user. LOL, see, told you I couldn't guest the word. :-D "predictor" is way better, thanks!
© 2016 - 2026 Red Hat, Inc.