Context switch and TLB flush support for processes that use a global
ASID & PCID across all CPUs.
At both context switch time and TLB flush time, we need to check
whether a task is switching to a global ASID, and reload the TLB
with the new ASID as appropriate.
In both code paths, we also short-circuit the TLB flush if we
are using a global ASID, because the global ASIDs are always
kept up to date across CPUs, even while the process is not
running on a CPU.
Signed-off-by: Rik van Riel <riel@surriel.com>
---
arch/x86/include/asm/tlbflush.h | 13 ++++++
arch/x86/mm/tlb.c | 77 ++++++++++++++++++++++++++++++---
2 files changed, 83 insertions(+), 7 deletions(-)
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 83f1da2f1e4a..f1f82571249b 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -240,6 +240,19 @@ static inline bool is_dyn_asid(u16 asid)
return asid < TLB_NR_DYN_ASIDS;
}
+static inline bool is_global_asid(u16 asid)
+{
+ return !is_dyn_asid(asid);
+}
+
+static inline bool in_asid_transition(struct mm_struct *mm)
+{
+ if (!cpu_feature_enabled(X86_FEATURE_INVLPGB))
+ return false;
+
+ return mm && READ_ONCE(mm->context.asid_transition);
+}
+
#ifdef CONFIG_X86_BROADCAST_TLB_FLUSH
static inline u16 mm_global_asid(struct mm_struct *mm)
{
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 405630479b90..d8a04e398615 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -227,6 +227,20 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen,
return;
}
+ /*
+ * TLB consistency for global ASIDs is maintained with hardware assisted
+ * remote TLB flushing. Global ASIDs are always up to date.
+ */
+ if (static_cpu_has(X86_FEATURE_INVLPGB)) {
+ u16 global_asid = mm_global_asid(next);
+
+ if (global_asid) {
+ *new_asid = global_asid;
+ *need_flush = false;
+ return;
+ }
+ }
+
if (this_cpu_read(cpu_tlbstate.invalidate_other))
clear_asid_other();
@@ -389,6 +403,23 @@ void destroy_context_free_global_asid(struct mm_struct *mm)
global_asid_available++;
}
+/*
+ * Is the mm transitioning from a CPU-local ASID to a global ASID?
+ */
+static bool needs_global_asid_reload(struct mm_struct *next, u16 prev_asid)
+{
+ u16 global_asid = mm_global_asid(next);
+
+ if (!static_cpu_has(X86_FEATURE_INVLPGB))
+ return false;
+
+ /* Process is transitioning to a global ASID */
+ if (global_asid && prev_asid != global_asid)
+ return true;
+
+ return false;
+}
+
/*
* Given an ASID, flush the corresponding user ASID. We can delay this
* until the next time we switch to it.
@@ -694,7 +725,8 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
*/
if (prev == next) {
/* Not actually switching mm's */
- VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) !=
+ VM_WARN_ON(is_dyn_asid(prev_asid) &&
+ this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) !=
next->context.ctx_id);
/*
@@ -711,6 +743,20 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
!cpumask_test_cpu(cpu, mm_cpumask(next))))
cpumask_set_cpu(cpu, mm_cpumask(next));
+ /* Check if the current mm is transitioning to a global ASID */
+ if (needs_global_asid_reload(next, prev_asid)) {
+ next_tlb_gen = atomic64_read(&next->context.tlb_gen);
+ choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush);
+ goto reload_tlb;
+ }
+
+ /*
+ * Broadcast TLB invalidation keeps this PCID up to date
+ * all the time.
+ */
+ if (is_global_asid(prev_asid))
+ return;
+
/*
* If the CPU is not in lazy TLB mode, we are just switching
* from one thread in a process to another thread in the same
@@ -744,6 +790,13 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
*/
cond_mitigation(tsk);
+ /*
+ * Let nmi_uaccess_okay() and finish_asid_transition()
+ * know that we're changing CR3.
+ */
+ this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING);
+ barrier();
+
/*
* Leave this CPU in prev's mm_cpumask. Atomic writes to
* mm_cpumask can be expensive under contention. The CPU
@@ -758,14 +811,12 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
next_tlb_gen = atomic64_read(&next->context.tlb_gen);
choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush);
-
- /* Let nmi_uaccess_okay() know that we're changing CR3. */
- this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING);
- barrier();
}
+reload_tlb:
new_lam = mm_lam_cr3_mask(next);
if (need_flush) {
+ VM_WARN_ON_ONCE(is_global_asid(new_asid));
this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id);
this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen);
load_new_mm_cr3(next->pgd, new_asid, new_lam, true);
@@ -884,7 +935,7 @@ static void flush_tlb_func(void *info)
const struct flush_tlb_info *f = info;
struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
- u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen);
+ u64 local_tlb_gen;
bool local = smp_processor_id() == f->initiating_cpu;
unsigned long nr_invalidate = 0;
u64 mm_tlb_gen;
@@ -907,6 +958,16 @@ static void flush_tlb_func(void *info)
if (unlikely(loaded_mm == &init_mm))
return;
+ /* Reload the ASID if transitioning into or out of a global ASID */
+ if (needs_global_asid_reload(loaded_mm, loaded_mm_asid)) {
+ switch_mm_irqs_off(NULL, loaded_mm, NULL);
+ loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
+ }
+
+ /* Broadcast ASIDs are always kept up to date with INVLPGB. */
+ if (is_global_asid(loaded_mm_asid))
+ return;
+
VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].ctx_id) !=
loaded_mm->context.ctx_id);
@@ -924,6 +985,8 @@ static void flush_tlb_func(void *info)
return;
}
+ local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen);
+
if (unlikely(f->new_tlb_gen != TLB_GENERATION_INVALID &&
f->new_tlb_gen <= local_tlb_gen)) {
/*
@@ -1091,7 +1154,7 @@ STATIC_NOPV void native_flush_tlb_multi(const struct cpumask *cpumask,
* up on the new contents of what used to be page tables, while
* doing a speculative memory access.
*/
- if (info->freed_tables)
+ if (info->freed_tables || in_asid_transition(info->mm))
on_each_cpu_mask(cpumask, flush_tlb_func, (void *)info, true);
else
on_each_cpu_cond_mask(should_flush_tlb, flush_tlb_func,
--
2.47.1Hi Rik,
kernel test robot noticed the following build errors:
[auto build test ERROR on tip/x86/core]
[also build test ERROR on tip/x86/mm tip/master linus/master v6.14-rc4 next-20250224]
[cannot apply to tip/auto-latest]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Rik-van-Riel/x86-mm-consolidate-full-flush-threshold-decision/20250224-035335
base: tip/x86/core
patch link: https://lore.kernel.org/r/20250223194943.3518952-9-riel%40surriel.com
patch subject: [PATCH v13 08/14] x86/mm: global ASID context switch & TLB flush handling
config: i386-buildonly-randconfig-001-20250224 (https://download.01.org/0day-ci/archive/20250225/202502250255.UQONr93N-lkp@intel.com/config)
compiler: clang version 19.1.3 (https://github.com/llvm/llvm-project ab51eccf88f5321e7c60591c5546b254b6afab99)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250225/202502250255.UQONr93N-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202502250255.UQONr93N-lkp@intel.com/
All errors (new ones prefixed by >>):
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:37: error: no member named 'asid_transition' in 'mm_context_t'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~~~~~~~~~~ ^
include/asm-generic/rwonce.h:49:33: note: expanded from macro 'READ_ONCE'
49 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:35: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:509:10: note: expanded from macro '__native_word'
509 | (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || \
| ^
include/linux/compiler_types.h:542:22: note: expanded from macro 'compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:530:23: note: expanded from macro '_compiletime_assert'
530 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:522:9: note: expanded from macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:37: error: no member named 'asid_transition' in 'mm_context_t'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~~~~~~~~~~ ^
include/asm-generic/rwonce.h:49:33: note: expanded from macro 'READ_ONCE'
49 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:35: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:509:39: note: expanded from macro '__native_word'
509 | (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || \
| ^
include/linux/compiler_types.h:542:22: note: expanded from macro 'compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:530:23: note: expanded from macro '_compiletime_assert'
530 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:522:9: note: expanded from macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:37: error: no member named 'asid_transition' in 'mm_context_t'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~~~~~~~~~~ ^
include/asm-generic/rwonce.h:49:33: note: expanded from macro 'READ_ONCE'
49 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:35: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:510:10: note: expanded from macro '__native_word'
510 | sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
| ^
include/linux/compiler_types.h:542:22: note: expanded from macro 'compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:530:23: note: expanded from macro '_compiletime_assert'
530 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:522:9: note: expanded from macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:37: error: no member named 'asid_transition' in 'mm_context_t'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~~~~~~~~~~ ^
include/asm-generic/rwonce.h:49:33: note: expanded from macro 'READ_ONCE'
49 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:35: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:510:38: note: expanded from macro '__native_word'
510 | sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
| ^
include/linux/compiler_types.h:542:22: note: expanded from macro 'compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:530:23: note: expanded from macro '_compiletime_assert'
530 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:522:9: note: expanded from macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:37: error: no member named 'asid_transition' in 'mm_context_t'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~~~~~~~~~~ ^
include/asm-generic/rwonce.h:49:33: note: expanded from macro 'READ_ONCE'
49 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:48: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:542:22: note: expanded from macro 'compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:530:23: note: expanded from macro '_compiletime_assert'
530 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:522:9: note: expanded from macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:37: error: no member named 'asid_transition' in 'mm_context_t'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~~~~~~~~~~ ^
include/asm-generic/rwonce.h:50:14: note: expanded from macro 'READ_ONCE'
50 | __READ_ONCE(x); \
| ^
include/asm-generic/rwonce.h:44:65: note: expanded from macro '__READ_ONCE'
44 | #define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x))
| ^
include/linux/compiler_types.h:498:13: note: expanded from macro '__unqual_scalar_typeof'
498 | _Generic((x), \
| ^
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:37: error: no member named 'asid_transition' in 'mm_context_t'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~~~~~~~~~~ ^
include/asm-generic/rwonce.h:50:14: note: expanded from macro 'READ_ONCE'
50 | __READ_ONCE(x); \
| ^
include/asm-generic/rwonce.h:44:65: note: expanded from macro '__READ_ONCE'
44 | #define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x))
| ^
include/linux/compiler_types.h:505:15: note: expanded from macro '__unqual_scalar_typeof'
505 | default: (x)))
| ^
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:37: error: no member named 'asid_transition' in 'mm_context_t'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~~~~~~~~~~ ^
include/asm-generic/rwonce.h:50:14: note: expanded from macro 'READ_ONCE'
50 | __READ_ONCE(x); \
| ^
include/asm-generic/rwonce.h:44:72: note: expanded from macro '__READ_ONCE'
44 | #define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x))
| ^
In file included from arch/x86/kernel/asm-offsets.c:14:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:13:
In file included from include/linux/cgroup.h:17:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:13:
In file included from include/linux/uaccess.h:12:
In file included from arch/x86/include/asm/uaccess.h:17:
>> arch/x86/include/asm/tlbflush.h:253:12: error: invalid operands to binary expression ('struct mm_struct *' and 'void')
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
9 errors generated.
make[3]: *** [scripts/Makefile.build:102: arch/x86/kernel/asm-offsets.s] Error 1 shuffle=1461763656
make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1264: prepare0] Error 2 shuffle=1461763656
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:251: __sub-make] Error 2 shuffle=1461763656
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:251: __sub-make] Error 2 shuffle=1461763656
make: Target 'prepare' not remade because of errors.
vim +253 arch/x86/include/asm/tlbflush.h
247
248 static inline bool in_asid_transition(struct mm_struct *mm)
249 {
250 if (!cpu_feature_enabled(X86_FEATURE_INVLPGB))
251 return false;
252
> 253 return mm && READ_ONCE(mm->context.asid_transition);
254 }
255
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Rik,
kernel test robot noticed the following build errors:
[auto build test ERROR on tip/x86/core]
[also build test ERROR on tip/x86/mm tip/master linus/master v6.14-rc4 next-20250221]
[cannot apply to tip/auto-latest]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Rik-van-Riel/x86-mm-consolidate-full-flush-threshold-decision/20250224-035335
base: tip/x86/core
patch link: https://lore.kernel.org/r/20250223194943.3518952-9-riel%40surriel.com
patch subject: [PATCH v13 08/14] x86/mm: global ASID context switch & TLB flush handling
config: x86_64-buildonly-randconfig-004-20250224 (https://download.01.org/0day-ci/archive/20250224/202502240650.kzshiji7-lkp@intel.com/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250224/202502240650.kzshiji7-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202502240650.kzshiji7-lkp@intel.com/
All errors (new ones prefixed by >>):
In file included from <command-line>:
arch/x86/include/asm/tlbflush.h: In function 'in_asid_transition':
>> arch/x86/include/asm/tlbflush.h:253:43: error: 'mm_context_t' has no member named 'asid_transition'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^
include/linux/compiler_types.h:522:23: note: in definition of macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:542:9: note: in expansion of macro '_compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:28: note: in expansion of macro '__native_word'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~
include/asm-generic/rwonce.h:49:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
49 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/x86/include/asm/tlbflush.h:253:22: note: in expansion of macro 'READ_ONCE'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^~~~~~~~~
>> arch/x86/include/asm/tlbflush.h:253:43: error: 'mm_context_t' has no member named 'asid_transition'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^
include/linux/compiler_types.h:522:23: note: in definition of macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:542:9: note: in expansion of macro '_compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:28: note: in expansion of macro '__native_word'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~
include/asm-generic/rwonce.h:49:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
49 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/x86/include/asm/tlbflush.h:253:22: note: in expansion of macro 'READ_ONCE'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^~~~~~~~~
>> arch/x86/include/asm/tlbflush.h:253:43: error: 'mm_context_t' has no member named 'asid_transition'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^
include/linux/compiler_types.h:522:23: note: in definition of macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:542:9: note: in expansion of macro '_compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:28: note: in expansion of macro '__native_word'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~
include/asm-generic/rwonce.h:49:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
49 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/x86/include/asm/tlbflush.h:253:22: note: in expansion of macro 'READ_ONCE'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^~~~~~~~~
>> arch/x86/include/asm/tlbflush.h:253:43: error: 'mm_context_t' has no member named 'asid_transition'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^
include/linux/compiler_types.h:522:23: note: in definition of macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:542:9: note: in expansion of macro '_compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:28: note: in expansion of macro '__native_word'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~
include/asm-generic/rwonce.h:49:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
49 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/x86/include/asm/tlbflush.h:253:22: note: in expansion of macro 'READ_ONCE'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^~~~~~~~~
>> arch/x86/include/asm/tlbflush.h:253:43: error: 'mm_context_t' has no member named 'asid_transition'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^
include/linux/compiler_types.h:522:23: note: in definition of macro '__compiletime_assert'
522 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:542:9: note: in expansion of macro '_compiletime_assert'
542 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:49:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
49 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/x86/include/asm/tlbflush.h:253:22: note: in expansion of macro 'READ_ONCE'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^~~~~~~~~
>> arch/x86/include/asm/tlbflush.h:253:43: error: 'mm_context_t' has no member named 'asid_transition'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^
include/linux/compiler_types.h:498:27: note: in definition of macro '__unqual_scalar_typeof'
498 | _Generic((x), \
| ^
include/asm-generic/rwonce.h:50:9: note: in expansion of macro '__READ_ONCE'
50 | __READ_ONCE(x); \
| ^~~~~~~~~~~
arch/x86/include/asm/tlbflush.h:253:22: note: in expansion of macro 'READ_ONCE'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^~~~~~~~~
In file included from ./arch/x86/include/generated/asm/rwonce.h:1,
from include/linux/compiler.h:354,
from include/linux/build_bug.h:5,
from include/linux/container_of.h:5,
from include/linux/list.h:5,
from include/linux/swait.h:5,
from include/linux/completion.h:12,
from include/linux/crypto.h:15,
from arch/x86/kernel/asm-offsets.c:9:
>> arch/x86/include/asm/tlbflush.h:253:43: error: 'mm_context_t' has no member named 'asid_transition'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^
include/asm-generic/rwonce.h:44:73: note: in definition of macro '__READ_ONCE'
44 | #define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x))
| ^
arch/x86/include/asm/tlbflush.h:253:22: note: in expansion of macro 'READ_ONCE'
253 | return mm && READ_ONCE(mm->context.asid_transition);
| ^~~~~~~~~
make[3]: *** [scripts/Makefile.build:102: arch/x86/kernel/asm-offsets.s] Error 1 shuffle=3847572294
make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1264: prepare0] Error 2 shuffle=3847572294
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:251: __sub-make] Error 2 shuffle=3847572294
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:251: __sub-make] Error 2 shuffle=3847572294
make: Target 'prepare' not remade because of errors.
vim +253 arch/x86/include/asm/tlbflush.h
247
248 static inline bool in_asid_transition(struct mm_struct *mm)
249 {
250 if (!cpu_feature_enabled(X86_FEATURE_INVLPGB))
251 return false;
252
> 253 return mm && READ_ONCE(mm->context.asid_transition);
254 }
255
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
On Mon, 24 Feb 2025 07:08:49 +0800
kernel test robot <lkp@intel.com> wrote:
> Hi Rik,
>
> kernel test robot noticed the following build errors:
>
> [auto build test ERROR on tip/x86/core]
> [also build test ERROR on tip/x86/mm tip/master linus/master v6.14-rc4 next-20250221]
This version of patch 8 fixes these compile errors.
I hoped the compiler was smart enough to elide the code when
broadcast TLB invalidation was disabled at the config level,
but maybe that happens at a later stage in the compilation,
after it's already thrown the errors...
I'm not entirely happy with this, but it does seem simple enough.
---8<---
x86/mm: global ASID context switch & TLB flush handling
Context switch and TLB flush support for processes that use a global
ASID & PCID across all CPUs.
At both context switch time and TLB flush time, we need to check
whether a task is switching to a global ASID, and reload the TLB
with the new ASID as appropriate.
In both code paths, we also short-circuit the TLB flush if we
are using a global ASID, because the global ASIDs are always
kept up to date across CPUs, even while the process is not
running on a CPU.
Signed-off-by: Rik van Riel <riel@surriel.com>
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 83f1da2f1e4a..24e6531e0b1a 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -240,6 +240,11 @@ static inline bool is_dyn_asid(u16 asid)
return asid < TLB_NR_DYN_ASIDS;
}
+static inline bool is_global_asid(u16 asid)
+{
+ return !is_dyn_asid(asid);
+}
+
#ifdef CONFIG_X86_BROADCAST_TLB_FLUSH
static inline u16 mm_global_asid(struct mm_struct *mm)
{
@@ -266,6 +271,14 @@ static inline void assign_mm_global_asid(struct mm_struct *mm, u16 asid)
mm->context.asid_transition = true;
smp_store_release(&mm->context.global_asid, asid);
}
+
+static inline bool in_asid_transition(struct mm_struct *mm)
+{
+ if (!cpu_feature_enabled(X86_FEATURE_INVLPGB))
+ return false;
+
+ return mm && READ_ONCE(mm->context.asid_transition);
+}
#else
static inline u16 mm_global_asid(struct mm_struct *mm)
{
@@ -275,6 +288,11 @@ static inline u16 mm_global_asid(struct mm_struct *mm)
static inline void assign_mm_global_asid(struct mm_struct *mm, u16 asid)
{
}
+
+static inline bool in_asid_transition(struct mm_struct *mm)
+{
+ return false;
+}
#endif
#ifdef CONFIG_PARAVIRT
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 405630479b90..e4aecd38ac4f 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -227,6 +227,20 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen,
return;
}
+ /*
+ * TLB consistency for global ASIDs is maintained with hardware assisted
+ * remote TLB flushing. Global ASIDs are always up to date.
+ */
+ if (static_cpu_has(X86_FEATURE_INVLPGB)) {
+ u16 global_asid = mm_global_asid(next);
+
+ if (global_asid) {
+ *new_asid = global_asid;
+ *need_flush = false;
+ return;
+ }
+ }
+
if (this_cpu_read(cpu_tlbstate.invalidate_other))
clear_asid_other();
@@ -382,11 +396,30 @@ void destroy_context_free_global_asid(struct mm_struct *mm)
guard(raw_spinlock_irqsave)(&global_asid_lock);
+#ifdef CONFIG_X86_BROADCAST_TLB_FLUSH
/* The global ASID can be re-used only after flush at wrap-around. */
__set_bit(mm->context.global_asid, global_asid_freed);
mm->context.global_asid = 0;
global_asid_available++;
+#endif
+}
+
+/*
+ * Is the mm transitioning from a CPU-local ASID to a global ASID?
+ */
+static bool needs_global_asid_reload(struct mm_struct *next, u16 prev_asid)
+{
+ u16 global_asid = mm_global_asid(next);
+
+ if (!static_cpu_has(X86_FEATURE_INVLPGB))
+ return false;
+
+ /* Process is transitioning to a global ASID */
+ if (global_asid && prev_asid != global_asid)
+ return true;
+
+ return false;
}
/*
@@ -694,7 +727,8 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
*/
if (prev == next) {
/* Not actually switching mm's */
- VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) !=
+ VM_WARN_ON(is_dyn_asid(prev_asid) &&
+ this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) !=
next->context.ctx_id);
/*
@@ -711,6 +745,20 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
!cpumask_test_cpu(cpu, mm_cpumask(next))))
cpumask_set_cpu(cpu, mm_cpumask(next));
+ /* Check if the current mm is transitioning to a global ASID */
+ if (needs_global_asid_reload(next, prev_asid)) {
+ next_tlb_gen = atomic64_read(&next->context.tlb_gen);
+ choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush);
+ goto reload_tlb;
+ }
+
+ /*
+ * Broadcast TLB invalidation keeps this PCID up to date
+ * all the time.
+ */
+ if (is_global_asid(prev_asid))
+ return;
+
/*
* If the CPU is not in lazy TLB mode, we are just switching
* from one thread in a process to another thread in the same
@@ -744,6 +792,13 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
*/
cond_mitigation(tsk);
+ /*
+ * Let nmi_uaccess_okay() and finish_asid_transition()
+ * know that we're changing CR3.
+ */
+ this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING);
+ barrier();
+
/*
* Leave this CPU in prev's mm_cpumask. Atomic writes to
* mm_cpumask can be expensive under contention. The CPU
@@ -758,14 +813,12 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
next_tlb_gen = atomic64_read(&next->context.tlb_gen);
choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush);
-
- /* Let nmi_uaccess_okay() know that we're changing CR3. */
- this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING);
- barrier();
}
+reload_tlb:
new_lam = mm_lam_cr3_mask(next);
if (need_flush) {
+ VM_WARN_ON_ONCE(is_global_asid(new_asid));
this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id);
this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen);
load_new_mm_cr3(next->pgd, new_asid, new_lam, true);
@@ -884,7 +937,7 @@ static void flush_tlb_func(void *info)
const struct flush_tlb_info *f = info;
struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
- u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen);
+ u64 local_tlb_gen;
bool local = smp_processor_id() == f->initiating_cpu;
unsigned long nr_invalidate = 0;
u64 mm_tlb_gen;
@@ -907,6 +960,16 @@ static void flush_tlb_func(void *info)
if (unlikely(loaded_mm == &init_mm))
return;
+ /* Reload the ASID if transitioning into or out of a global ASID */
+ if (needs_global_asid_reload(loaded_mm, loaded_mm_asid)) {
+ switch_mm_irqs_off(NULL, loaded_mm, NULL);
+ loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
+ }
+
+ /* Broadcast ASIDs are always kept up to date with INVLPGB. */
+ if (is_global_asid(loaded_mm_asid))
+ return;
+
VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].ctx_id) !=
loaded_mm->context.ctx_id);
@@ -924,6 +987,8 @@ static void flush_tlb_func(void *info)
return;
}
+ local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen);
+
if (unlikely(f->new_tlb_gen != TLB_GENERATION_INVALID &&
f->new_tlb_gen <= local_tlb_gen)) {
/*
@@ -1091,7 +1156,7 @@ STATIC_NOPV void native_flush_tlb_multi(const struct cpumask *cpumask,
* up on the new contents of what used to be page tables, while
* doing a speculative memory access.
*/
- if (info->freed_tables)
+ if (info->freed_tables || in_asid_transition(info->mm))
on_each_cpu_mask(cpumask, flush_tlb_func, (void *)info, true);
else
on_each_cpu_cond_mask(should_flush_tlb, flush_tlb_func,
On Mon, 2025-02-24 at 07:08 +0800, kernel test robot wrote: > > All errors (new ones prefixed by >>): > > In file included from <command-line>: > arch/x86/include/asm/tlbflush.h: In function 'in_asid_transition': > > > arch/x86/include/asm/tlbflush.h:253:43: error: 'mm_context_t' has > > > no member named 'asid_transition' > 253 | return mm && READ_ONCE(mm- > >context.asid_transition); > | ^ Looks like this one needs to be moved under the #ifdef, too. Short-circuiting the build by hard-disabling X86_FEATURE_INVLPGB, or even adding an explicit if(!IS_ENABLED(CONFIG_X86_BROADCAST_TLB_FLUSH)) is not enough to get the compiler to short-circuit the rest of the code in that function. We might need an #ifdef inside destroy_context_free_global_asid as well, no good way to move that one into a .h file without making the global asid data structures in tlb.c visible to the entire kernel instead of keeping them static in tlb.c -- All Rights Reversed.
© 2016 - 2025 Red Hat, Inc.