From: Kevin Loughlin <kevinloughlin@google.com>
In line with WBINVD usage, add WBONINVD helper functions. Explicitly fall
back to WBINVD (via alternative()) if WBNOINVD isn't supported even though
the instruction itself is backwards compatible (WBNOINVD is WBINVD with an
ignored REP prefix), so that disabling X86_FEATURE_WBNOINVD behaves as one
would expect, e.g. in case there's a hardware issue that affects WBNOINVD.
Opportunsitically add comments explaining the architectural behavior of
WBINVD and WBNOINVD, and to provide hints and pointers to uarch-specific
behavior.
Note, alternative() ensures compatibility with early boot code as needed.
Signed-off-by: Kevin Loughlin <kevinloughlin@google.com>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: Kai Huang <kai.huang@intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Co-developed-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
---
arch/x86/include/asm/smp.h | 6 ++++++
arch/x86/include/asm/special_insns.h | 32 +++++++++++++++++++++++++++-
arch/x86/lib/cache-smp.c | 11 ++++++++++
3 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 028f126018c9..e08f1ae25401 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -113,6 +113,7 @@ void native_play_dead(void);
void play_dead_common(void);
void wbinvd_on_cpu(int cpu);
void wbinvd_on_all_cpus(void);
+void wbnoinvd_on_all_cpus(void);
void smp_kick_mwait_play_dead(void);
void __noreturn mwait_play_dead(unsigned int eax_hint);
@@ -153,6 +154,11 @@ static inline void wbinvd_on_all_cpus(void)
wbinvd();
}
+static inline void wbnoinvd_on_all_cpus(void)
+{
+ wbnoinvd();
+}
+
static inline struct cpumask *cpu_llc_shared_mask(int cpu)
{
return (struct cpumask *)cpumask_of(0);
diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h
index 6266d6b9e0b8..f2240c4ac0ea 100644
--- a/arch/x86/include/asm/special_insns.h
+++ b/arch/x86/include/asm/special_insns.h
@@ -115,9 +115,39 @@ static inline void wrpkru(u32 pkru)
}
#endif
+/*
+ * Write back all modified lines in all levels of cache associated with this
+ * logical processor to main memory, and then invalidate all caches. Depending
+ * on the micro-architecture, WBINVD (and WBNOINVD below) may or may not affect
+ * lower level caches associated with another logical processor that shares any
+ * level of this processor’s cache hierarchy.
+ *
+ * Note, AMD CPUs enumerate the behavior or WB{NO}{INVD} with respect to other
+ * logical, non-originating processors in CPUID 0x8000001D.EAX[N:0].
+ */
static __always_inline void wbinvd(void)
{
- asm volatile("wbinvd": : :"memory");
+ asm volatile("wbinvd" : : : "memory");
+}
+
+/* Instruction encoding provided for binutils backwards compatibility. */
+#define ASM_WBNOINVD _ASM_BYTES(0xf3,0x0f,0x09)
+
+/*
+ * Write back all modified lines in all levels of cache associated with this
+ * logical processor to main memory, but do NOT explicitly invalidate caches,
+ * i.e. leave all/most cache lines in the hierarchy in non-modified state.
+ */
+static __always_inline void wbnoinvd(void)
+{
+ /*
+ * Explicitly encode WBINVD if X86_FEATURE_WBNOINVD is unavailable even
+ * though WBNOINVD is backwards compatible (it's simply WBINVD with an
+ * ignored REP prefix), to guarantee that WBNOINVD isn't used if it
+ * needs to be avoided for any reason. For all supported usage in the
+ * kernel, WBINVD is functionally a superset of WBNOINVD.
+ */
+ alternative("wbinvd", ASM_WBNOINVD, X86_FEATURE_WBNOINVD);
}
static inline unsigned long __read_cr4(void)
diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
index 079c3f3cd32c..1789db5d8825 100644
--- a/arch/x86/lib/cache-smp.c
+++ b/arch/x86/lib/cache-smp.c
@@ -19,3 +19,14 @@ void wbinvd_on_all_cpus(void)
on_each_cpu(__wbinvd, NULL, 1);
}
EXPORT_SYMBOL(wbinvd_on_all_cpus);
+
+static void __wbnoinvd(void *dummy)
+{
+ wbnoinvd();
+}
+
+void wbnoinvd_on_all_cpus(void)
+{
+ on_each_cpu(__wbnoinvd, NULL, 1);
+}
+EXPORT_SYMBOL(wbnoinvd_on_all_cpus);
--
2.49.0.1151.ga128411c76-googOn Thu, May 22, 2025 at 04:37:27PM -0700, Sean Christopherson wrote:
> diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> index 079c3f3cd32c..1789db5d8825 100644
> --- a/arch/x86/lib/cache-smp.c
> +++ b/arch/x86/lib/cache-smp.c
> @@ -19,3 +19,14 @@ void wbinvd_on_all_cpus(void)
> on_each_cpu(__wbinvd, NULL, 1);
> }
> EXPORT_SYMBOL(wbinvd_on_all_cpus);
> +
> +static void __wbnoinvd(void *dummy)
> +{
> + wbnoinvd();
> +}
> +
> +void wbnoinvd_on_all_cpus(void)
> +{
> + on_each_cpu(__wbnoinvd, NULL, 1);
> +}
> +EXPORT_SYMBOL(wbnoinvd_on_all_cpus);
If there's no particular reason for the non-GPL export besides being
consistent with the rest - yes, I did the change for wbinvd_on_all_cpus() but
that was loooong time ago - I'd simply make this export _GPL.
Thx.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
On Thu, Jul 10, 2025 at 01:29:02PM +0200, Borislav Petkov wrote:
> On Thu, May 22, 2025 at 04:37:27PM -0700, Sean Christopherson wrote:
> > diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> > index 079c3f3cd32c..1789db5d8825 100644
> > --- a/arch/x86/lib/cache-smp.c
> > +++ b/arch/x86/lib/cache-smp.c
> > @@ -19,3 +19,14 @@ void wbinvd_on_all_cpus(void)
> > on_each_cpu(__wbinvd, NULL, 1);
> > }
> > EXPORT_SYMBOL(wbinvd_on_all_cpus);
> > +
> > +static void __wbnoinvd(void *dummy)
> > +{
> > + wbnoinvd();
> > +}
> > +
> > +void wbnoinvd_on_all_cpus(void)
> > +{
> > + on_each_cpu(__wbnoinvd, NULL, 1);
> > +}
> > +EXPORT_SYMBOL(wbnoinvd_on_all_cpus);
>
> If there's no particular reason for the non-GPL export besides being
> consistent with the rest - yes, I did the change for wbinvd_on_all_cpus() but
> that was loooong time ago - I'd simply make this export _GPL.
Uhhhh, how about we use this fancy export to known modules only thing
for this?
These are typical things we do *NOT* want people to actually use.
On Thu, Jul 10, 2025 at 04:37:29PM +0200, Peter Zijlstra wrote:
> On Thu, Jul 10, 2025 at 01:29:02PM +0200, Borislav Petkov wrote:
> > On Thu, May 22, 2025 at 04:37:27PM -0700, Sean Christopherson wrote:
> > > diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> > > index 079c3f3cd32c..1789db5d8825 100644
> > > --- a/arch/x86/lib/cache-smp.c
> > > +++ b/arch/x86/lib/cache-smp.c
> > > @@ -19,3 +19,14 @@ void wbinvd_on_all_cpus(void)
> > > on_each_cpu(__wbinvd, NULL, 1);
> > > }
> > > EXPORT_SYMBOL(wbinvd_on_all_cpus);
> > > +
> > > +static void __wbnoinvd(void *dummy)
> > > +{
> > > + wbnoinvd();
> > > +}
> > > +
> > > +void wbnoinvd_on_all_cpus(void)
> > > +{
> > > + on_each_cpu(__wbnoinvd, NULL, 1);
> > > +}
> > > +EXPORT_SYMBOL(wbnoinvd_on_all_cpus);
> >
> > If there's no particular reason for the non-GPL export besides being
> > consistent with the rest - yes, I did the change for wbinvd_on_all_cpus() but
> > that was loooong time ago - I'd simply make this export _GPL.
>
> Uhhhh, how about we use this fancy export to known modules only thing
> for this?
>
> These are typical things we do *NOT* want people to actually use.
So kvm-amd is the SEV stuff, AGPGART is the ancient crap nobody cares
about, CCP is more SEV stuff, DRM actually does CLFLUSH loops, but has a
WBINVD fallback. i915 is rude and actually does WBINVD. Could they
pretty please also do CLFLUSH loops?
Anyway, the below seems to survive an allmodconfig.
---
diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
index c5c60d07308c..ac3cc32a4054 100644
--- a/arch/x86/lib/cache-smp.c
+++ b/arch/x86/lib/cache-smp.c
@@ -12,19 +12,19 @@ void wbinvd_on_cpu(int cpu)
{
smp_call_function_single(cpu, __wbinvd, NULL, 1);
}
-EXPORT_SYMBOL(wbinvd_on_cpu);
+EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_cpu, "kvm-amd,agpgart,ccp,drm,i915");
void wbinvd_on_all_cpus(void)
{
on_each_cpu(__wbinvd, NULL, 1);
}
-EXPORT_SYMBOL(wbinvd_on_all_cpus);
+EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_all_cpus, "kvm-amd,agpgart,ccp,drm,i915,intel-gtt");
void wbinvd_on_cpus_mask(struct cpumask *cpus)
{
on_each_cpu_mask(cpus, __wbinvd, NULL, 1);
}
-EXPORT_SYMBOL_GPL(wbinvd_on_cpus_mask);
+EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_cpus_mask, "kvm,kvm-amd,agpgart,ccp,drm,i915");
static void __wbnoinvd(void *dummy)
{
@@ -35,10 +35,10 @@ void wbnoinvd_on_all_cpus(void)
{
on_each_cpu(__wbnoinvd, NULL, 1);
}
-EXPORT_SYMBOL_GPL(wbnoinvd_on_all_cpus);
+EXPORT_SYMBOL_GPL_FOR_MODULES(wbnoinvd_on_all_cpus, "kvm-amd,agpgart,ccp,drm,i915");
void wbnoinvd_on_cpus_mask(struct cpumask *cpus)
{
on_each_cpu_mask(cpus, __wbnoinvd, NULL, 1);
}
-EXPORT_SYMBOL_GPL(wbnoinvd_on_cpus_mask);
+EXPORT_SYMBOL_GPL_FOR_MODULES(wbnoinvd_on_cpus_mask, "kvm-amd,agpgart,ccp,drm,i915");
On Thu, Jul 10, 2025 at 05:47:05PM +0200, Peter Zijlstra wrote: > So kvm-amd is the SEV stuff, AGPGART is the ancient crap nobody cares > about, CCP is more SEV stuff, DRM actually does CLFLUSH loops, but has a > WBINVD fallback. i915 is rude and actually does WBINVD. Could they > pretty please also do CLFLUSH loops? So having looked at i915 a little more: drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c: * Currently we just do a heavy handed wbinvd_on_all_cpus() here since drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c: wbinvd_on_all_cpus(); This one can be runtime; but is only applied on certain parts that might be less than coherent -- hopefully not new parts, but who knows. Parts not taking this path end up calling drm_clflush_sg(), which DTRT. And these all look suspend/resume related, so we can live with them: drivers/gpu/drm/i915/gem/i915_gem_pm.c:#define wbinvd_on_all_cpus() \ drivers/gpu/drm/i915/gem/i915_gem_pm.c: wbinvd_on_all_cpus(); drivers/gpu/drm/i915/gem/i915_gem_pm.c: wbinvd_on_all_cpus(); drivers/gpu/drm/i915/gt/intel_ggtt.c: wbinvd_on_all_cpus();
On Thu, Jul 10, 2025 at 05:47:04PM +0200, Peter Zijlstra wrote:
> On Thu, Jul 10, 2025 at 04:37:29PM +0200, Peter Zijlstra wrote:
> > On Thu, Jul 10, 2025 at 01:29:02PM +0200, Borislav Petkov wrote:
> > > On Thu, May 22, 2025 at 04:37:27PM -0700, Sean Christopherson wrote:
> > > > diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> > > > index 079c3f3cd32c..1789db5d8825 100644
> > > > --- a/arch/x86/lib/cache-smp.c
> > > > +++ b/arch/x86/lib/cache-smp.c
> > > > @@ -19,3 +19,14 @@ void wbinvd_on_all_cpus(void)
> > > > on_each_cpu(__wbinvd, NULL, 1);
> > > > }
> > > > EXPORT_SYMBOL(wbinvd_on_all_cpus);
> > > > +
> > > > +static void __wbnoinvd(void *dummy)
> > > > +{
> > > > + wbnoinvd();
> > > > +}
> > > > +
> > > > +void wbnoinvd_on_all_cpus(void)
> > > > +{
> > > > + on_each_cpu(__wbnoinvd, NULL, 1);
> > > > +}
> > > > +EXPORT_SYMBOL(wbnoinvd_on_all_cpus);
> > >
> > > If there's no particular reason for the non-GPL export besides being
> > > consistent with the rest - yes, I did the change for wbinvd_on_all_cpus() but
> > > that was loooong time ago - I'd simply make this export _GPL.
> >
> > Uhhhh, how about we use this fancy export to known modules only thing
> > for this?
> >
> > These are typical things we do *NOT* want people to actually use.
>
> So kvm-amd is the SEV stuff, AGPGART is the ancient crap nobody cares
> about, CCP is more SEV stuff, DRM actually does CLFLUSH loops, but has a
> WBINVD fallback. i915 is rude and actually does WBINVD. Could they
> pretty please also do CLFLUSH loops?
There's some ancient pentium M where clflush is a no-op (or at least not
getting stuff flushed enough for the gpu to see it), but we still need to
ensure cache coherency with the non-coherent gpu, and wbinvd does the job.
I figured this out with sheer desperation over a decade ago myself,
it's pain.
There shouldn't be any other reason for i915 to wbinvd.
-Sima
>
> Anyway, the below seems to survive an allmodconfig.
>
> ---
> diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> index c5c60d07308c..ac3cc32a4054 100644
> --- a/arch/x86/lib/cache-smp.c
> +++ b/arch/x86/lib/cache-smp.c
> @@ -12,19 +12,19 @@ void wbinvd_on_cpu(int cpu)
> {
> smp_call_function_single(cpu, __wbinvd, NULL, 1);
> }
> -EXPORT_SYMBOL(wbinvd_on_cpu);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_cpu, "kvm-amd,agpgart,ccp,drm,i915");
>
> void wbinvd_on_all_cpus(void)
> {
> on_each_cpu(__wbinvd, NULL, 1);
> }
> -EXPORT_SYMBOL(wbinvd_on_all_cpus);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_all_cpus, "kvm-amd,agpgart,ccp,drm,i915,intel-gtt");
>
> void wbinvd_on_cpus_mask(struct cpumask *cpus)
> {
> on_each_cpu_mask(cpus, __wbinvd, NULL, 1);
> }
> -EXPORT_SYMBOL_GPL(wbinvd_on_cpus_mask);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_cpus_mask, "kvm,kvm-amd,agpgart,ccp,drm,i915");
>
> static void __wbnoinvd(void *dummy)
> {
> @@ -35,10 +35,10 @@ void wbnoinvd_on_all_cpus(void)
> {
> on_each_cpu(__wbnoinvd, NULL, 1);
> }
> -EXPORT_SYMBOL_GPL(wbnoinvd_on_all_cpus);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbnoinvd_on_all_cpus, "kvm-amd,agpgart,ccp,drm,i915");
>
> void wbnoinvd_on_cpus_mask(struct cpumask *cpus)
> {
> on_each_cpu_mask(cpus, __wbnoinvd, NULL, 1);
> }
> -EXPORT_SYMBOL_GPL(wbnoinvd_on_cpus_mask);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbnoinvd_on_cpus_mask, "kvm-amd,agpgart,ccp,drm,i915");
--
Simona Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
On Thu, Jul 10, 2025, Peter Zijlstra wrote:
> On Thu, Jul 10, 2025 at 04:37:29PM +0200, Peter Zijlstra wrote:
> > On Thu, Jul 10, 2025 at 01:29:02PM +0200, Borislav Petkov wrote:
> > > On Thu, May 22, 2025 at 04:37:27PM -0700, Sean Christopherson wrote:
> > > > diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> > > > index 079c3f3cd32c..1789db5d8825 100644
> > > > --- a/arch/x86/lib/cache-smp.c
> > > > +++ b/arch/x86/lib/cache-smp.c
> > > > @@ -19,3 +19,14 @@ void wbinvd_on_all_cpus(void)
> > > > on_each_cpu(__wbinvd, NULL, 1);
> > > > }
> > > > EXPORT_SYMBOL(wbinvd_on_all_cpus);
> > > > +
> > > > +static void __wbnoinvd(void *dummy)
> > > > +{
> > > > + wbnoinvd();
> > > > +}
> > > > +
> > > > +void wbnoinvd_on_all_cpus(void)
> > > > +{
> > > > + on_each_cpu(__wbnoinvd, NULL, 1);
> > > > +}
> > > > +EXPORT_SYMBOL(wbnoinvd_on_all_cpus);
> > >
> > > If there's no particular reason for the non-GPL export besides being
> > > consistent with the rest - yes, I did the change for wbinvd_on_all_cpus() but
> > > that was loooong time ago - I'd simply make this export _GPL.
> >
> > Uhhhh, how about we use this fancy export to known modules only thing
> > for this?
> >
> > These are typical things we do *NOT* want people to actually use.
>
> So kvm-amd is the SEV stuff, AGPGART is the ancient crap nobody cares
> about, CCP is more SEV stuff, DRM actually does CLFLUSH loops, but has a
> WBINVD fallback. i915 is rude and actually does WBINVD. Could they
> pretty please also do CLFLUSH loops?
FWIW, doing CLFLUSH in KVM isn't feasible. In multiple flows, KVM doesn't have
a valid virtual mapping, and hardware *requires* a WBINVD for at least one of the
SEV paths.
> Anyway, the below seems to survive an allmodconfig.
>
> ---
> diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> index c5c60d07308c..ac3cc32a4054 100644
> --- a/arch/x86/lib/cache-smp.c
> +++ b/arch/x86/lib/cache-smp.c
> @@ -12,19 +12,19 @@ void wbinvd_on_cpu(int cpu)
> {
> smp_call_function_single(cpu, __wbinvd, NULL, 1);
> }
> -EXPORT_SYMBOL(wbinvd_on_cpu);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_cpu, "kvm-amd,agpgart,ccp,drm,i915");
Patch 5 of this series drops KVM's homebrewed version of WBINVD-on-CPU, so this
one at least would need to export the symbol for "kvm" as well.
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 1d0e9180148d..d63a2c27e058 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4965,11 +4965,6 @@ long kvm_arch_dev_ioctl(struct file *filp,
return r;
}
-static void wbinvd_ipi(void *garbage)
-{
- wbinvd();
-}
-
static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
return kvm_arch_has_noncoherent_dma(vcpu->kvm);
@@ -4991,8 +4986,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
if (kvm_x86_call(has_wbinvd_exit)())
cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
- smp_call_function_single(vcpu->cpu,
- wbinvd_ipi, NULL, 1);
+ wbinvd_on_cpu(vcpu->cpu);
}
kvm_x86_call(vcpu_load)(vcpu, cpu);
--
>
> void wbinvd_on_all_cpus(void)
> {
> on_each_cpu(__wbinvd, NULL, 1);
> }
> -EXPORT_SYMBOL(wbinvd_on_all_cpus);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_all_cpus, "kvm-amd,agpgart,ccp,drm,i915,intel-gtt");
>
> void wbinvd_on_cpus_mask(struct cpumask *cpus)
> {
> on_each_cpu_mask(cpus, __wbinvd, NULL, 1);
> }
> -EXPORT_SYMBOL_GPL(wbinvd_on_cpus_mask);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_cpus_mask, "kvm,kvm-amd,agpgart,ccp,drm,i915");
>
> static void __wbnoinvd(void *dummy)
> {
> @@ -35,10 +35,10 @@ void wbnoinvd_on_all_cpus(void)
> {
> on_each_cpu(__wbnoinvd, NULL, 1);
> }
> -EXPORT_SYMBOL_GPL(wbnoinvd_on_all_cpus);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbnoinvd_on_all_cpus, "kvm-amd,agpgart,ccp,drm,i915");
>
> void wbnoinvd_on_cpus_mask(struct cpumask *cpus)
> {
> on_each_cpu_mask(cpus, __wbnoinvd, NULL, 1);
> }
> -EXPORT_SYMBOL_GPL(wbnoinvd_on_cpus_mask);
> +EXPORT_SYMBOL_GPL_FOR_MODULES(wbnoinvd_on_cpus_mask, "kvm-amd,agpgart,ccp,drm,i915");
On Thu, Jul 10, 2025 at 08:55:02AM -0700, Sean Christopherson wrote:
> > So kvm-amd is the SEV stuff, AGPGART is the ancient crap nobody cares
> > about, CCP is more SEV stuff, DRM actually does CLFLUSH loops, but has a
> > WBINVD fallback. i915 is rude and actually does WBINVD. Could they
> > pretty please also do CLFLUSH loops?
>
> FWIW, doing CLFLUSH in KVM isn't feasible. In multiple flows, KVM doesn't have
> a valid virtual mapping, and hardware *requires* a WBINVD for at least one of the
> SEV paths.
Yeah, I know. We should give the hardware folks more grief about this
though. If we ever get into the situation of requiring WBINVD, they've
messed up.
> > Anyway, the below seems to survive an allmodconfig.
> >
> > ---
> > diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> > index c5c60d07308c..ac3cc32a4054 100644
> > --- a/arch/x86/lib/cache-smp.c
> > +++ b/arch/x86/lib/cache-smp.c
> > @@ -12,19 +12,19 @@ void wbinvd_on_cpu(int cpu)
> > {
> > smp_call_function_single(cpu, __wbinvd, NULL, 1);
> > }
> > -EXPORT_SYMBOL(wbinvd_on_cpu);
> > +EXPORT_SYMBOL_GPL_FOR_MODULES(wbinvd_on_cpu, "kvm-amd,agpgart,ccp,drm,i915");
>
> Patch 5 of this series drops KVM's homebrewed version of WBINVD-on-CPU, so this
> one at least would need to export the symbol for "kvm" as well.
Ah, sure. At the same time, cpumask_of(cpu) is very cheap.
On Thu, Jul 10, 2025, Borislav Petkov wrote:
> On Thu, May 22, 2025 at 04:37:27PM -0700, Sean Christopherson wrote:
> > diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
> > index 079c3f3cd32c..1789db5d8825 100644
> > --- a/arch/x86/lib/cache-smp.c
> > +++ b/arch/x86/lib/cache-smp.c
> > @@ -19,3 +19,14 @@ void wbinvd_on_all_cpus(void)
> > on_each_cpu(__wbinvd, NULL, 1);
> > }
> > EXPORT_SYMBOL(wbinvd_on_all_cpus);
> > +
> > +static void __wbnoinvd(void *dummy)
> > +{
> > + wbnoinvd();
> > +}
> > +
> > +void wbnoinvd_on_all_cpus(void)
> > +{
> > + on_each_cpu(__wbnoinvd, NULL, 1);
> > +}
> > +EXPORT_SYMBOL(wbnoinvd_on_all_cpus);
>
> If there's no particular reason for the non-GPL export besides being
> consistent with the rest - yes, I did the change for wbinvd_on_all_cpus() but
> that was loooong time ago - I'd simply make this export _GPL.
Yeah, AFAIK, no reason other than consistency. GPL it.
On Thu, Jul 10, 2025 at 06:56:38AM -0700, Sean Christopherson wrote:
> Yeah, AFAIK, no reason other than consistency. GPL it.
Done. Tag x86_core_for_kvm on tip.
It should appear on the mirrors soon, I hope.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 07f99c3fbe6e322bdb222fbfd59f708ced799cc5
Gitweb: https://git.kernel.org/tip/07f99c3fbe6e322bdb222fbfd59f708ced799cc5
Author: Kevin Loughlin <kevinloughlin@google.com>
AuthorDate: Thu, 22 May 2025 16:37:27 -07:00
Committer: Borislav Petkov (AMD) <bp@alien8.de>
CommitterDate: Thu, 10 Jul 2025 13:23:10 +02:00
x86/lib: Add WBNOINVD helper functions
In line with WBINVD usage, add WBNOINVD helper functions. Explicitly fall
back to WBINVD (via alternative()) if WBNOINVD isn't supported even though
the instruction itself is backwards compatible (WBNOINVD is WBINVD with an
ignored REP prefix), so that disabling X86_FEATURE_WBNOINVD behaves as one
would expect, e.g. in case there's a hardware issue that affects WBNOINVD.
Opportunistically, add comments explaining the architectural behavior of
WBINVD and WBNOINVD, and provide hints and pointers to uarch-specific
behavior.
Note, alternative() ensures compatibility with early boot code as needed.
[ bp: Massage, fix typos, make export _GPL. ]
Signed-off-by: Kevin Loughlin <kevinloughlin@google.com>
Co-developed-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: Kai Huang <kai.huang@intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/20250522233733.3176144-4-seanjc@google.com
---
arch/x86/include/asm/smp.h | 6 ++++++-
arch/x86/include/asm/special_insns.h | 29 ++++++++++++++++++++++++++-
arch/x86/lib/cache-smp.c | 11 ++++++++++-
3 files changed, 45 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 028f126..e08f1ae 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -113,6 +113,7 @@ void native_play_dead(void);
void play_dead_common(void);
void wbinvd_on_cpu(int cpu);
void wbinvd_on_all_cpus(void);
+void wbnoinvd_on_all_cpus(void);
void smp_kick_mwait_play_dead(void);
void __noreturn mwait_play_dead(unsigned int eax_hint);
@@ -153,6 +154,11 @@ static inline void wbinvd_on_all_cpus(void)
wbinvd();
}
+static inline void wbnoinvd_on_all_cpus(void)
+{
+ wbnoinvd();
+}
+
static inline struct cpumask *cpu_llc_shared_mask(int cpu)
{
return (struct cpumask *)cpumask_of(0);
diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h
index ecda17e..fde2bd7 100644
--- a/arch/x86/include/asm/special_insns.h
+++ b/arch/x86/include/asm/special_insns.h
@@ -104,9 +104,36 @@ static inline void wrpkru(u32 pkru)
}
#endif
+/*
+ * Write back all modified lines in all levels of cache associated with this
+ * logical processor to main memory, and then invalidate all caches. Depending
+ * on the micro-architecture, WBINVD (and WBNOINVD below) may or may not affect
+ * lower level caches associated with another logical processor that shares any
+ * level of this processor's cache hierarchy.
+ */
static __always_inline void wbinvd(void)
{
- asm volatile("wbinvd": : :"memory");
+ asm volatile("wbinvd" : : : "memory");
+}
+
+/* Instruction encoding provided for binutils backwards compatibility. */
+#define ASM_WBNOINVD _ASM_BYTES(0xf3,0x0f,0x09)
+
+/*
+ * Write back all modified lines in all levels of cache associated with this
+ * logical processor to main memory, but do NOT explicitly invalidate caches,
+ * i.e. leave all/most cache lines in the hierarchy in non-modified state.
+ */
+static __always_inline void wbnoinvd(void)
+{
+ /*
+ * Explicitly encode WBINVD if X86_FEATURE_WBNOINVD is unavailable even
+ * though WBNOINVD is backwards compatible (it's simply WBINVD with an
+ * ignored REP prefix), to guarantee that WBNOINVD isn't used if it
+ * needs to be avoided for any reason. For all supported usage in the
+ * kernel, WBINVD is functionally a superset of WBNOINVD.
+ */
+ alternative("wbinvd", ASM_WBNOINVD, X86_FEATURE_WBNOINVD);
}
static inline unsigned long __read_cr4(void)
diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c
index 079c3f3..74e0d5b 100644
--- a/arch/x86/lib/cache-smp.c
+++ b/arch/x86/lib/cache-smp.c
@@ -19,3 +19,14 @@ void wbinvd_on_all_cpus(void)
on_each_cpu(__wbinvd, NULL, 1);
}
EXPORT_SYMBOL(wbinvd_on_all_cpus);
+
+static void __wbnoinvd(void *dummy)
+{
+ wbnoinvd();
+}
+
+void wbnoinvd_on_all_cpus(void)
+{
+ on_each_cpu(__wbnoinvd, NULL, 1);
+}
+EXPORT_SYMBOL_GPL(wbnoinvd_on_all_cpus);© 2016 - 2025 Red Hat, Inc.