arch/x86/include/asm/fpu/sched.h | 5 +++-- arch/x86/kernel/fpu/core.c | 33 +++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-)
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 450d73dc5f9188ade87fe2d2d967cf18e63b280f
Gitweb: https://git.kernel.org/tip/450d73dc5f9188ade87fe2d2d967cf18e63b280f
Author: Dapeng Mi <dapeng1.mi@linux.intel.com>
AuthorDate: Mon, 24 Aug 2026 16:27:15 +08:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Wed, 02 Sep 2026 13:10:41 +02:00
x86/fpu: Add update_fpu_state_and_flag() helper
Add update_fpu_state_and_flag() as suggested by Peter and Dave.
The helper saves user FPU state and then sets TIF_NEED_FPU_LOAD,
ensuring the task FPU state is saved whenever the flag is set.
Subsequent patches will use this guarantee in NMI context by checking
TIF_NEED_FPU_LOAD before retrieving user FPU state from the saved
task FPU state.
Also add barrier() in the host/guest FPU state switch path and move
clearing of fpu->__task_fpstate after switching back to host state. So
fpu->__task_fpstate is always observed as host FPU state when non-NULL.
Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/all/20251204154721.GB2619703@noisy.programming.kicks-ass.net/
Link: https://patch.msgid.link/20260824082731.1013973-8-dapeng1.mi@linux.intel.com
---
arch/x86/include/asm/fpu/sched.h | 5 +++--
arch/x86/kernel/fpu/core.c | 33 +++++++++++++++++++++++++------
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h
index 89004f4..dcb2fa5 100644
--- a/arch/x86/include/asm/fpu/sched.h
+++ b/arch/x86/include/asm/fpu/sched.h
@@ -10,6 +10,8 @@
#include <asm/trace/fpu.h>
extern void save_fpregs_to_fpstate(struct fpu *fpu);
+extern void update_fpu_state_and_flag(struct fpu *fpu,
+ struct task_struct *task);
extern void fpu__drop(struct task_struct *tsk);
extern int fpu_clone(struct task_struct *dst, u64 clone_flags, bool minimal,
unsigned long shstk_addr);
@@ -36,8 +38,7 @@ static inline void switch_fpu(struct task_struct *old, int cpu)
!(old->flags & (PF_KTHREAD | PF_USER_WORKER))) {
struct fpu *old_fpu = x86_task_fpu(old);
- set_tsk_thread_flag(old, TIF_NEED_FPU_LOAD);
- save_fpregs_to_fpstate(old_fpu);
+ update_fpu_state_and_flag(old_fpu, old);
/*
* The save operation preserved register state, so the
* fpu_fpregs_owner_ctx is still @old_fpu. Store the
diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index d1aeecd..047244e 100644
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -213,6 +213,19 @@ void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask)
}
}
+/*
+ * Save the FPU register state in fpu->fpstate->regs and set
+ * TIF_NEED_FPU_LOAD subsequently.
+ *
+ * Must be called with fpregs_lock() held, ensuring flag
+ * TIF_NEED_FPU_LOAD is set last.
+ */
+void update_fpu_state_and_flag(struct fpu *fpu, struct task_struct *task)
+{
+ save_fpregs_to_fpstate(fpu);
+ set_tsk_thread_flag(task, TIF_NEED_FPU_LOAD);
+}
+
void fpu_reset_from_exception_fixup(void)
{
restore_fpregs_from_fpstate(&init_fpstate, XFEATURE_MASK_FPSTATE);
@@ -383,13 +396,13 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest)
/* Swap fpstate */
if (enter_guest) {
- fpu->__task_fpstate = cur_fps;
+ WRITE_ONCE(fpu->__task_fpstate, cur_fps);
+ barrier();
fpu->fpstate = guest_fps;
guest_fps->in_use = true;
} else {
guest_fps->in_use = false;
fpu->fpstate = fpu->__task_fpstate;
- fpu->__task_fpstate = NULL;
}
cur_fps = fpu->fpstate;
@@ -406,6 +419,16 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest)
xfd_update_state(cur_fps);
}
+ /*
+ * Clear fpu->__task_fpstate after switching back to host state.
+ * A non-NULL __task_fpstate means guest state is still resident in
+ * hardware; reset it only once host state has been restored.
+ */
+ if (!enter_guest) {
+ barrier();
+ WRITE_ONCE(fpu->__task_fpstate, NULL);
+ }
+
fpregs_mark_activate();
fpregs_unlock();
return 0;
@@ -481,10 +504,8 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask)
this_cpu_write(kernel_fpu_allowed, false);
if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) &&
- !test_thread_flag(TIF_NEED_FPU_LOAD)) {
- set_thread_flag(TIF_NEED_FPU_LOAD);
- save_fpregs_to_fpstate(x86_task_fpu(current));
- }
+ !test_thread_flag(TIF_NEED_FPU_LOAD))
+ update_fpu_state_and_flag(x86_task_fpu(current), current);
__cpu_invalidate_fpregs_state();
/* Put sane initial values into the control registers. */© 2016 - 2026 Red Hat, Inc.