[v4] Intel RAR TLB invalidation

[RFC PATCH v4 6/8] x86/mm: use RAR for kernel TLB flushes

Posted by Rik van Riel 3 months, 3 weeks ago

From: Rik van Riel <riel@fb.com>

Use Intel RAR for kernel TLB flushes, when enabled.

Pass in PCID 0 to smp_call_rar_many() to flush the specified addresses,
regardless of which PCID they might be cached under in any destination CPU.

Signed-off-by: Rik van Riel <riel@surriel.com>
---
 arch/x86/mm/tlb.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 39f80111e6f1..8931f7029d6c 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -21,6 +21,7 @@
 #include <asm/apic.h>
 #include <asm/msr.h>
 #include <asm/perf_event.h>
+#include <asm/rar.h>
 #include <asm/tlb.h>
 
 #include "mm_internal.h"
@@ -1468,6 +1469,18 @@ static void do_flush_tlb_all(void *info)
 	__flush_tlb_all();
 }
 
+static void rar_full_flush(const cpumask_t *cpumask)
+{
+	guard(preempt)();
+	smp_call_rar_many(cpumask, 0, 0, TLB_FLUSH_ALL);
+	invpcid_flush_all();
+}
+
+static void rar_flush_all(void)
+{
+	rar_full_flush(cpu_online_mask);
+}
+
 void flush_tlb_all(void)
 {
 	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
@@ -1475,6 +1488,8 @@ void flush_tlb_all(void)
 	/* First try (faster) hardware-assisted TLB invalidation. */
 	if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
 		invlpgb_flush_all();
+	else if (cpu_feature_enabled(X86_FEATURE_RAR))
+		rar_flush_all();
 	else
 		/* Fall back to the IPI-based invalidation. */
 		on_each_cpu(do_flush_tlb_all, NULL, 1);
@@ -1504,15 +1519,36 @@ static void do_kernel_range_flush(void *info)
 	struct flush_tlb_info *f = info;
 	unsigned long addr;
 
+	/*
+	 * With PTI kernel TLB entries in all PCIDs need to be flushed.
+	 * With RAR the PCID space becomes so large, we might as well flush it all.
+	 *
+	 * Either of the two by itself works with targeted flushes.
+	 */
+	if (cpu_feature_enabled(X86_FEATURE_RAR) &&
+	    cpu_feature_enabled(X86_FEATURE_PTI)) {
+		invpcid_flush_all();
+		return;
+	}
+
 	/* flush range by one by one 'invlpg' */
 	for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
 		flush_tlb_one_kernel(addr);
 }
 
+static void rar_kernel_range_flush(struct flush_tlb_info *info)
+{
+	guard(preempt)();
+	smp_call_rar_many(cpu_online_mask, 0, info->start, info->end);
+	do_kernel_range_flush(info);
+}
+
 static void kernel_tlb_flush_all(struct flush_tlb_info *info)
 {
 	if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
 		invlpgb_flush_all();
+	else if (cpu_feature_enabled(X86_FEATURE_RAR))
+		rar_flush_all();
 	else
 		on_each_cpu(do_flush_tlb_all, NULL, 1);
 }
@@ -1521,6 +1557,8 @@ static void kernel_tlb_flush_range(struct flush_tlb_info *info)
 {
 	if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
 		invlpgb_kernel_range_flush(info);
+	else if (cpu_feature_enabled(X86_FEATURE_RAR))
+		rar_kernel_range_flush(info);
 	else
 		on_each_cpu(do_kernel_range_flush, info, 1);
 }
-- 
2.49.0

Re: [RFC PATCH v4 6/8] x86/mm: use RAR for kernel TLB flushes

Posted by Kirill A. Shutemov 3 months, 1 week ago

On Thu, Jun 19, 2025 at 04:03:58PM -0400, Rik van Riel wrote:
> From: Rik van Riel <riel@fb.com>
> 
> Use Intel RAR for kernel TLB flushes, when enabled.
> 
> Pass in PCID 0 to smp_call_rar_many() to flush the specified addresses,
> regardless of which PCID they might be cached under in any destination CPU.
> 
> Signed-off-by: Rik van Riel <riel@surriel.com>
> ---
>  arch/x86/mm/tlb.c | 38 ++++++++++++++++++++++++++++++++++++++
>  1 file changed, 38 insertions(+)
> 
> diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
> index 39f80111e6f1..8931f7029d6c 100644
> --- a/arch/x86/mm/tlb.c
> +++ b/arch/x86/mm/tlb.c
> @@ -21,6 +21,7 @@
>  #include <asm/apic.h>
>  #include <asm/msr.h>
>  #include <asm/perf_event.h>
> +#include <asm/rar.h>
>  #include <asm/tlb.h>
>  
>  #include "mm_internal.h"
> @@ -1468,6 +1469,18 @@ static void do_flush_tlb_all(void *info)
>  	__flush_tlb_all();
>  }
>  
> +static void rar_full_flush(const cpumask_t *cpumask)
> +{
> +	guard(preempt)();
> +	smp_call_rar_many(cpumask, 0, 0, TLB_FLUSH_ALL);
> +	invpcid_flush_all();

I don't follow why do we need to call invpcid_flush_all() here in addition
to smp_call_rar_many(). Hm?

-- 
  Kiryl Shutsemau / Kirill A. Shutemov

Re: [RFC PATCH v4 6/8] x86/mm: use RAR for kernel TLB flushes

Posted by Rik van Riel 3 months, 1 week ago

On Fri, 2025-06-27 at 16:27 +0300, Kirill A. Shutemov wrote:
> On Thu, Jun 19, 2025 at 04:03:58PM -0400, Rik van Riel wrote:
> > From: Rik van Riel <riel@fb.com>
> > 
> > Use Intel RAR for kernel TLB flushes, when enabled.
> > 
> > Pass in PCID 0 to smp_call_rar_many() to flush the specified
> > addresses,
> > regardless of which PCID they might be cached under in any
> > destination CPU.
> > 
> > Signed-off-by: Rik van Riel <riel@surriel.com>
> > ---
> >  arch/x86/mm/tlb.c | 38 ++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 38 insertions(+)
> > 
> > diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
> > index 39f80111e6f1..8931f7029d6c 100644
> > --- a/arch/x86/mm/tlb.c
> > +++ b/arch/x86/mm/tlb.c
> > @@ -21,6 +21,7 @@
> >  #include <asm/apic.h>
> >  #include <asm/msr.h>
> >  #include <asm/perf_event.h>
> > +#include <asm/rar.h>
> >  #include <asm/tlb.h>
> >  
> >  #include "mm_internal.h"
> > @@ -1468,6 +1469,18 @@ static void do_flush_tlb_all(void *info)
> >  	__flush_tlb_all();
> >  }
> >  
> > +static void rar_full_flush(const cpumask_t *cpumask)
> > +{
> > +	guard(preempt)();
> > +	smp_call_rar_many(cpumask, 0, 0, TLB_FLUSH_ALL);
> > +	invpcid_flush_all();
> 
> I don't follow why do we need to call invpcid_flush_all() here in
> addition
> to smp_call_rar_many(). Hm?
> 
We shouldn't have to.

Once we figure out why the RAR flush isn't
working right (despite the RAR transitioning
from RAR_PENDING to RAR_SUCCESS) we should be
able to get rid of this call.


-- 
All Rights Reversed.