arch/x86/kernel/fpu/xstate.h | 54 ++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 24 deletions(-)
Rewrite XSTATE_{XSAVE,XRESTORE} macros as static inline functions
in order to use asm goto to optimize exception handling.
The code that used XSTATE_XSAVE improves from:
3f1fe: 49 0f ae 64 24 40 xsave64 0x40(%r12)
3f204: 31 ed xor %ebp,%ebp
3f206: 85 ed test %ebp,%ebp
3f208: 75 37 jne 3f241 <...>
to just:
3f201: 48 0f ae 65 40 xsave64 0x40(%rbp)
because there is no need to set and test temporary result
variable anymore.
bloat-o-meter reports a small code size improvement
(x86_64 defconfig, gcc-14.2.1):
add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-25 (-25)
Function old new delta
--------------------------------------------------------
save_fpregs_to_fpstate 141 135 -6
__fpu_restore_sig 1448 1429 -19
Total: Before=22809695, After=22809670, chg -0.00%
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
---
arch/x86/kernel/fpu/xstate.h | 54 ++++++++++++++++++++----------------
1 file changed, 30 insertions(+), 24 deletions(-)
diff --git a/arch/x86/kernel/fpu/xstate.h b/arch/x86/kernel/fpu/xstate.h
index 1418423bc4c9..c8e31005567a 100644
--- a/arch/x86/kernel/fpu/xstate.h
+++ b/arch/x86/kernel/fpu/xstate.h
@@ -131,32 +131,38 @@ static inline int update_pkru_in_sigframe(struct xregs_state __user *buf, u64 ma
*
* Use XSAVE as a fallback.
*/
-#define XSTATE_XSAVE(st, lmask, hmask, err) \
- asm volatile("1: " ALTERNATIVE_3(XSAVE, \
- XSAVEOPT, X86_FEATURE_XSAVEOPT, \
- XSAVEC, X86_FEATURE_XSAVEC, \
- XSAVES, X86_FEATURE_XSAVES) \
- "\n\t" \
- "xor %[err], %[err]\n" \
- "3:\n" \
- _ASM_EXTABLE_TYPE_REG(1b, 3b, EX_TYPE_EFAULT_REG, %[err]) \
- : [err] "=r" (err) \
- : [xa] "m" (*(st)), "a" (lmask), "d" (hmask) \
- : "memory")
+static __always_inline int __xstate_xsave(struct xregs_state *st,
+ u32 lmask, u32 hmask)
+{
+ asm goto("1: " ALTERNATIVE_3(XSAVE,
+ XSAVEOPT, X86_FEATURE_XSAVEOPT,
+ XSAVEC, X86_FEATURE_XSAVEC,
+ XSAVES, X86_FEATURE_XSAVES)
+ _ASM_EXTABLE(1b, %l[fault])
+ :
+ : [xa] "m" (*st), "a" (lmask), "d" (hmask)
+ : "memory"
+ : fault);
+ return 0;
+fault:
+ return -EFAULT;
+}
/*
* Use XRSTORS to restore context if it is enabled. XRSTORS supports compact
* XSAVE area format.
*/
-#define XSTATE_XRESTORE(st, lmask, hmask) \
- asm volatile("1: " ALTERNATIVE(XRSTOR, \
- XRSTORS, X86_FEATURE_XSAVES) \
- "\n" \
- "3:\n" \
- _ASM_EXTABLE_TYPE(1b, 3b, EX_TYPE_FPU_RESTORE) \
- : \
- : [xa] "m" (*(st)), "a" (lmask), "d" (hmask) \
- : "memory")
+static __always_inline void __xstate_xrstor(struct xregs_state *st,
+ u32 lmask, u32 hmask)
+{
+ asm volatile("1: " ALTERNATIVE(XRSTOR,
+ XRSTORS, X86_FEATURE_XSAVES)
+ "3:\n"
+ _ASM_EXTABLE_TYPE(1b, 3b, EX_TYPE_FPU_RESTORE)
+ :
+ : [xa] "m" (*st), "a" (lmask), "d" (hmask)
+ : "memory");
+}
#if defined(CONFIG_X86_64) && defined(CONFIG_X86_DEBUG_FPU)
extern void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor);
@@ -208,7 +214,7 @@ static inline void os_xsave(struct fpstate *fpstate)
WARN_ON_FPU(!alternatives_patched);
xfd_validate_state(fpstate, mask, false);
- XSTATE_XSAVE(&fpstate->regs.xsave, lmask, hmask, err);
+ err = __xstate_xsave(&fpstate->regs.xsave, lmask, hmask);
/* We should never fault when copying to a kernel buffer: */
WARN_ON_FPU(err);
@@ -225,7 +231,7 @@ static inline void os_xrstor(struct fpstate *fpstate, u64 mask)
u32 hmask = mask >> 32;
xfd_validate_state(fpstate, mask, true);
- XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
+ __xstate_xrstor(&fpstate->regs.xsave, lmask, hmask);
}
/* Restore of supervisor state. Does not require XFD */
@@ -235,7 +241,7 @@ static inline void os_xrstor_supervisor(struct fpstate *fpstate)
u32 lmask = mask;
u32 hmask = mask >> 32;
- XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
+ __xstate_xrstor(&fpstate->regs.xsave, lmask, hmask);
}
/*
--
2.48.1© 2016 - 2025 Red Hat, Inc.