Currently, x86, Riscv, Loongarch use the generic entry. Convert arm64
to use the generic entry infrastructure from kernel/entry/*. The generic
entry makes maintainers' work easier and codes more elegant, which also
removed duplicate 150+ LOC. The changes are below:
- Remove TIF_SYSCALL_* flag, _TIF_WORK_MASK, _TIF_SYSCALL_WORK
- Remove syscall_trace_enter/exit() and use generic one.
- Remove *enter_from/exit_to_kernel_mode(), and wrap with generic
irqentry_enter/exit().
- Remove *enter_from/exit_to_user_mode(), and wrap with generic
irqentry_enter_from/exit_to_user_mode().
- Remove arm64_enter/exit_nmi() and use generic irqentry_nmi_enter/exit().
- Remove PREEMPT_DYNAMIC code, as generic entry will do it ok by
implementing arch_irqentry_exit_need_resched().
Tested ok with following test cases on Qemu cortex-a53 and HiSilicon
Kunpeng-920:
- Run `perf top` command
- Switch between different `dynamic preempt` mode
- Use `pseudo nmi`
- stress-ng CPU stress test.
- MTE test case in Documentation/arch/arm64/memory-tagging-extension.rst
and all test cases in tools/testing/selftests/arm64/mte/* (Only Qemu).
Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com>
Tested-by: Jinjie Ruan <ruanjinjie@huawei.com>
Tested-by: Kees Cook <kees@kernel.org>
---
v3:
- Test the MTE test cases.
- Handle forget_syscall() in arch_post_report_syscall_entry()
- Rebased on the previous patch, so move the arch funcs to entry-common.h.
- Change __always_inline() to inline for arch_enter_from_kernel_mode() and
arch_exit_to_kernel_mode_prepare(), add inline for the other arch funcs.
- Remove unused signal.h for entry-common.h.
v2:
- Add Tested-by.
- Rebased on the refactored report_syscall() code.
- Add comment for arch_prepare_report_syscall_exit().
- Update the commit message.
- Adjust entry-common.h header file inclusion to alphabetical order.
---
arch/arm64/Kconfig | 1 +
arch/arm64/include/asm/entry-common.h | 172 ++++++++++++
arch/arm64/include/asm/syscall.h | 6 +-
arch/arm64/include/asm/thread_info.h | 23 +-
arch/arm64/kernel/entry-common.c | 368 +++++---------------------
arch/arm64/kernel/ptrace.c | 101 -------
arch/arm64/kernel/signal.c | 3 +-
arch/arm64/kernel/syscall.c | 18 +-
8 files changed, 264 insertions(+), 428 deletions(-)
create mode 100644 arch/arm64/include/asm/entry-common.h
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 5d91259ee7b5..e6ccc5ea06fe 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -138,6 +138,7 @@ config ARM64
select GENERIC_CPU_DEVICES
select GENERIC_CPU_VULNERABILITIES
select GENERIC_EARLY_IOREMAP
+ select GENERIC_ENTRY
select GENERIC_IDLE_POLL_SETUP
select GENERIC_IOREMAP
select GENERIC_IRQ_IPI
diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h
new file mode 100644
index 000000000000..a14d62b2eb69
--- /dev/null
+++ b/arch/arm64/include/asm/entry-common.h
@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _ASM_ARM64_ENTRY_COMMON_H
+#define _ASM_ARM64_ENTRY_COMMON_H
+
+#include <linux/thread_info.h>
+
+#include <asm/daifflags.h>
+#include <asm/fpsimd.h>
+#include <asm/mte.h>
+#include <asm/stacktrace.h>
+
+enum ptrace_syscall_dir {
+ PTRACE_SYSCALL_ENTER = 0,
+ PTRACE_SYSCALL_EXIT,
+};
+
+#define ARCH_EXIT_TO_USER_MODE_WORK (_TIF_MTE_ASYNC_FAULT | _TIF_FOREIGN_FPSTATE)
+
+static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs)
+{
+ mte_disable_tco_entry(current);
+}
+
+#define arch_enter_from_user_mode arch_enter_from_user_mode
+
+static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
+ unsigned long ti_work)
+{
+ if (ti_work & _TIF_MTE_ASYNC_FAULT) {
+ clear_thread_flag(TIF_MTE_ASYNC_FAULT);
+ send_sig_fault(SIGSEGV, SEGV_MTEAERR, (void __user *)NULL, current);
+ }
+
+ if (ti_work & _TIF_FOREIGN_FPSTATE)
+ fpsimd_restore_current_state();
+}
+
+#define arch_exit_to_user_mode_work arch_exit_to_user_mode_work
+
+static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
+ unsigned long ti_work)
+{
+ local_daif_mask();
+ mte_check_tfsr_exit();
+}
+
+#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
+
+static inline void arch_enter_from_kernel_mode(struct pt_regs *regs)
+{
+ mte_check_tfsr_entry();
+ mte_disable_tco_entry(current);
+}
+
+#define arch_enter_from_kernel_mode arch_enter_from_kernel_mode
+
+static inline void arch_exit_to_kernel_mode_prepare(struct pt_regs *regs)
+{
+ mte_check_tfsr_exit();
+}
+
+#define arch_exit_to_kernel_mode_prepare arch_exit_to_kernel_mode_prepare
+
+static inline bool arch_irqentry_exit_need_resched(void)
+{
+ /*
+ * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
+ * priority masking is used the GIC irqchip driver will clear DAIF.IF
+ * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
+ * DAIF we must have handled an NMI, so skip preemption.
+ */
+ if (system_uses_irq_prio_masking() && read_sysreg(daif))
+ return false;
+
+ /*
+ * Preempting a task from an IRQ means we leave copies of PSTATE
+ * on the stack. cpufeature's enable calls may modify PSTATE, but
+ * resuming one of these preempted tasks would undo those changes.
+ *
+ * Only allow a task to be preempted once cpufeatures have been
+ * enabled.
+ */
+ if (!system_capabilities_finalized())
+ return false;
+
+ return true;
+}
+
+#define arch_irqentry_exit_need_resched arch_irqentry_exit_need_resched
+
+static inline unsigned long arch_prepare_report_syscall_entry(struct pt_regs *regs)
+{
+ unsigned long saved_reg;
+ int regno;
+
+ /*
+ * We have some ABI weirdness here in the way that we handle syscall
+ * exit stops because we indicate whether or not the stop has been
+ * signalled from syscall entry or syscall exit by clobbering a general
+ * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
+ * and restoring its old value after the stop. This means that:
+ *
+ * - Any writes by the tracer to this register during the stop are
+ * ignored/discarded.
+ *
+ * - The actual value of the register is not available during the stop,
+ * so the tracer cannot save it and restore it later.
+ *
+ * - Syscall stops behave differently to seccomp and pseudo-step traps
+ * (the latter do not nobble any registers).
+ */
+ regno = (is_compat_task() ? 12 : 7);
+ saved_reg = regs->regs[regno];
+ regs->regs[regno] = PTRACE_SYSCALL_ENTER;
+
+ return saved_reg;
+}
+
+#define arch_prepare_report_syscall_entry arch_prepare_report_syscall_entry
+
+static inline void arch_post_report_syscall_entry(struct pt_regs *regs,
+ unsigned long saved_reg, long ret)
+{
+ int regno = (is_compat_task() ? 12 : 7);
+
+ if (ret)
+ forget_syscall(regs);
+
+ regs->regs[regno] = saved_reg;
+}
+
+#define arch_post_report_syscall_entry arch_post_report_syscall_entry
+
+static inline unsigned long arch_prepare_report_syscall_exit(struct pt_regs *regs,
+ unsigned long work)
+{
+ unsigned long saved_reg;
+ int regno;
+
+ /* See comment for arch_prepare_report_syscall_entry() */
+ regno = (is_compat_task() ? 12 : 7);
+ saved_reg = regs->regs[regno];
+ regs->regs[regno] = PTRACE_SYSCALL_EXIT;
+
+ if (report_single_step(work)) {
+ /*
+ * Signal a pseudo-step exception since we are stepping but
+ * tracer modifications to the registers may have rewound the
+ * state machine.
+ */
+ regs->regs[regno] = saved_reg;
+ }
+
+ return saved_reg;
+}
+
+#define arch_prepare_report_syscall_exit arch_prepare_report_syscall_exit
+
+static inline void arch_post_report_syscall_exit(struct pt_regs *regs,
+ unsigned long saved_reg,
+ unsigned long work)
+{
+ int regno = (is_compat_task() ? 12 : 7);
+
+ if (!report_single_step(work))
+ regs->regs[regno] = saved_reg;
+}
+
+#define arch_post_report_syscall_exit arch_post_report_syscall_exit
+
+#endif /* _ASM_ARM64_ENTRY_COMMON_H */
diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h
index ab8e14b96f68..9891b15da4c3 100644
--- a/arch/arm64/include/asm/syscall.h
+++ b/arch/arm64/include/asm/syscall.h
@@ -85,7 +85,9 @@ static inline int syscall_get_arch(struct task_struct *task)
return AUDIT_ARCH_AARCH64;
}
-int syscall_trace_enter(struct pt_regs *regs);
-void syscall_trace_exit(struct pt_regs *regs);
+static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
+{
+ return false;
+}
#endif /* __ASM_SYSCALL_H */
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index e72a3bf9e563..ec5d74c53bf9 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -43,6 +43,7 @@ struct thread_info {
void *scs_sp;
#endif
u32 cpu;
+ unsigned long syscall_work; /* SYSCALL_WORK_ flags */
};
#define thread_saved_pc(tsk) \
@@ -64,11 +65,6 @@ void arch_setup_new_exec(void);
#define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */
#define TIF_MTE_ASYNC_FAULT 5 /* MTE Asynchronous Tag Check Fault */
#define TIF_NOTIFY_SIGNAL 6 /* signal notifications exist */
-#define TIF_SYSCALL_TRACE 8 /* syscall trace active */
-#define TIF_SYSCALL_AUDIT 9 /* syscall auditing */
-#define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */
-#define TIF_SECCOMP 11 /* syscall secure computing */
-#define TIF_SYSCALL_EMU 12 /* syscall emulation active */
#define TIF_MEMDIE 18 /* is terminating due to OOM killer */
#define TIF_FREEZE 19
#define TIF_RESTORE_SIGMASK 20
@@ -86,27 +82,12 @@ void arch_setup_new_exec(void);
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
#define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)
-#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
-#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
-#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
-#define _TIF_SECCOMP (1 << TIF_SECCOMP)
-#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
-#define _TIF_UPROBE (1 << TIF_UPROBE)
-#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
+#define _TIF_UPROBE (1 << TIF_UPROBE)
#define _TIF_32BIT (1 << TIF_32BIT)
#define _TIF_SVE (1 << TIF_SVE)
#define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT)
#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
-#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
- _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
- _TIF_UPROBE | _TIF_MTE_ASYNC_FAULT | \
- _TIF_NOTIFY_SIGNAL)
-
-#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
- _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
- _TIF_SYSCALL_EMU)
-
#ifdef CONFIG_SHADOW_CALL_STACK
#define INIT_SCS \
.scs_base = init_shadow_call_stack, \
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index b77a15955f28..ac61aeb93933 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -14,6 +14,7 @@
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/thread_info.h>
+#include <linux/entry-common.h>
#include <asm/cpufeature.h>
#include <asm/daifflags.h>
@@ -28,201 +29,15 @@
#include <asm/sysreg.h>
#include <asm/system_misc.h>
-/*
- * Handle IRQ/context state management when entering from kernel mode.
- * Before this function is called it is not safe to call regular kernel code,
- * instrumentable code, or any code which may trigger an exception.
- *
- * This is intended to match the logic in irqentry_enter(), handling the kernel
- * mode transitions only.
- */
-static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)
-{
- regs->exit_rcu = false;
-
- if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
- lockdep_hardirqs_off(CALLER_ADDR0);
- ct_irq_enter();
- trace_hardirqs_off_finish();
-
- regs->exit_rcu = true;
- return;
- }
-
- lockdep_hardirqs_off(CALLER_ADDR0);
- rcu_irq_enter_check_tick();
- trace_hardirqs_off_finish();
-}
-
-static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
-{
- __enter_from_kernel_mode(regs);
- mte_check_tfsr_entry();
- mte_disable_tco_entry(current);
-}
-
-/*
- * Handle IRQ/context state management when exiting to kernel mode.
- * After this function returns it is not safe to call regular kernel code,
- * instrumentable code, or any code which may trigger an exception.
- *
- * This is intended to match the logic in irqentry_exit(), handling the kernel
- * mode transitions only, and with preemption handled elsewhere.
- */
-static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)
-{
- lockdep_assert_irqs_disabled();
-
- if (interrupts_enabled(regs)) {
- if (regs->exit_rcu) {
- trace_hardirqs_on_prepare();
- lockdep_hardirqs_on_prepare();
- ct_irq_exit();
- lockdep_hardirqs_on(CALLER_ADDR0);
- return;
- }
-
- trace_hardirqs_on();
- } else {
- if (regs->exit_rcu)
- ct_irq_exit();
- }
-}
-
-static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
-{
- mte_check_tfsr_exit();
- __exit_to_kernel_mode(regs);
-}
-
-/*
- * Handle IRQ/context state management when entering from user mode.
- * Before this function is called it is not safe to call regular kernel code,
- * instrumentable code, or any code which may trigger an exception.
- */
-static __always_inline void __enter_from_user_mode(void)
+static __always_inline void exit_to_user_mode_wrapper(struct pt_regs *regs)
{
- lockdep_hardirqs_off(CALLER_ADDR0);
- CT_WARN_ON(ct_state() != CONTEXT_USER);
- user_exit_irqoff();
- trace_hardirqs_off_finish();
- mte_disable_tco_entry(current);
-}
-
-static __always_inline void enter_from_user_mode(struct pt_regs *regs)
-{
- __enter_from_user_mode();
-}
-
-/*
- * Handle IRQ/context state management when exiting to user mode.
- * After this function returns it is not safe to call regular kernel code,
- * instrumentable code, or any code which may trigger an exception.
- */
-static __always_inline void __exit_to_user_mode(void)
-{
- trace_hardirqs_on_prepare();
- lockdep_hardirqs_on_prepare();
- user_enter_irqoff();
- lockdep_hardirqs_on(CALLER_ADDR0);
-}
-
-static void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags)
-{
- do {
- local_irq_enable();
-
- if (thread_flags & _TIF_NEED_RESCHED)
- schedule();
-
- if (thread_flags & _TIF_UPROBE)
- uprobe_notify_resume(regs);
-
- if (thread_flags & _TIF_MTE_ASYNC_FAULT) {
- clear_thread_flag(TIF_MTE_ASYNC_FAULT);
- send_sig_fault(SIGSEGV, SEGV_MTEAERR,
- (void __user *)NULL, current);
- }
-
- if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
- do_signal(regs);
-
- if (thread_flags & _TIF_NOTIFY_RESUME)
- resume_user_mode_work(regs);
-
- if (thread_flags & _TIF_FOREIGN_FPSTATE)
- fpsimd_restore_current_state();
-
- local_irq_disable();
- thread_flags = read_thread_flags();
- } while (thread_flags & _TIF_WORK_MASK);
-}
-
-static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)
-{
- unsigned long flags;
-
local_irq_disable();
-
- flags = read_thread_flags();
- if (unlikely(flags & _TIF_WORK_MASK))
- do_notify_resume(regs, flags);
-
- local_daif_mask();
-
- lockdep_sys_exit();
-}
-
-static __always_inline void exit_to_user_mode(struct pt_regs *regs)
-{
- exit_to_user_mode_prepare(regs);
- mte_check_tfsr_exit();
- __exit_to_user_mode();
+ irqentry_exit_to_user_mode(regs);
}
asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
{
- exit_to_user_mode(regs);
-}
-
-/*
- * Handle IRQ/context state management when entering an NMI from user/kernel
- * mode. Before this function is called it is not safe to call regular kernel
- * code, instrumentable code, or any code which may trigger an exception.
- */
-static void noinstr arm64_enter_nmi(struct pt_regs *regs)
-{
- regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
-
- __nmi_enter();
- lockdep_hardirqs_off(CALLER_ADDR0);
- lockdep_hardirq_enter();
- ct_nmi_enter();
-
- trace_hardirqs_off_finish();
- ftrace_nmi_enter();
-}
-
-/*
- * Handle IRQ/context state management when exiting an NMI from user/kernel
- * mode. After this function returns it is not safe to call regular kernel
- * code, instrumentable code, or any code which may trigger an exception.
- */
-static void noinstr arm64_exit_nmi(struct pt_regs *regs)
-{
- bool restore = regs->lockdep_hardirqs;
-
- ftrace_nmi_exit();
- if (restore) {
- trace_hardirqs_on_prepare();
- lockdep_hardirqs_on_prepare();
- }
-
- ct_nmi_exit();
- lockdep_hardirq_exit();
- if (restore)
- lockdep_hardirqs_on(CALLER_ADDR0);
- __nmi_exit();
+ exit_to_user_mode_wrapper(regs);
}
/*
@@ -259,48 +74,6 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
lockdep_hardirqs_on(CALLER_ADDR0);
}
-#ifdef CONFIG_PREEMPT_DYNAMIC
-DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
-#define need_irq_preemption() \
- (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
-#else
-#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION))
-#endif
-
-static void __sched arm64_preempt_schedule_irq(void)
-{
- if (!need_irq_preemption())
- return;
-
- /*
- * Note: thread_info::preempt_count includes both thread_info::count
- * and thread_info::need_resched, and is not equivalent to
- * preempt_count().
- */
- if (READ_ONCE(current_thread_info()->preempt_count) != 0)
- return;
-
- /*
- * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
- * priority masking is used the GIC irqchip driver will clear DAIF.IF
- * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
- * DAIF we must have handled an NMI, so skip preemption.
- */
- if (system_uses_irq_prio_masking() && read_sysreg(daif))
- return;
-
- /*
- * Preempting a task from an IRQ means we leave copies of PSTATE
- * on the stack. cpufeature's enable calls may modify PSTATE, but
- * resuming one of these preempted tasks would undo those changes.
- *
- * Only allow a task to be preempted once cpufeatures have been
- * enabled.
- */
- if (system_capabilities_finalized())
- preempt_schedule_irq();
-}
-
static void do_interrupt_handler(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
@@ -320,7 +93,7 @@ extern void (*handle_arch_fiq)(struct pt_regs *);
static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
unsigned long esr)
{
- arm64_enter_nmi(regs);
+ irqentry_nmi_enter(regs);
console_verbose();
@@ -426,41 +199,43 @@ UNHANDLED(el1t, 64, error)
static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
{
unsigned long far = read_sysreg(far_el1);
+ irqentry_state_t state = irqentry_enter(regs);
- enter_from_kernel_mode(regs);
local_daif_inherit(regs);
do_mem_abort(far, esr, regs);
local_daif_mask();
- exit_to_kernel_mode(regs);
+ irqentry_exit(regs, state);
}
static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
{
unsigned long far = read_sysreg(far_el1);
+ irqentry_state_t state = irqentry_enter(regs);
- enter_from_kernel_mode(regs);
local_daif_inherit(regs);
do_sp_pc_abort(far, esr, regs);
local_daif_mask();
- exit_to_kernel_mode(regs);
+ irqentry_exit(regs, state);
}
static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
{
- enter_from_kernel_mode(regs);
+ irqentry_state_t state = irqentry_enter(regs);
+
local_daif_inherit(regs);
do_el1_undef(regs, esr);
local_daif_mask();
- exit_to_kernel_mode(regs);
+ irqentry_exit(regs, state);
}
static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
{
- enter_from_kernel_mode(regs);
+ irqentry_state_t state = irqentry_enter(regs);
+
local_daif_inherit(regs);
do_el1_bti(regs, esr);
local_daif_mask();
- exit_to_kernel_mode(regs);
+ irqentry_exit(regs, state);
}
static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
@@ -475,11 +250,12 @@ static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
{
- enter_from_kernel_mode(regs);
+ irqentry_state_t state = irqentry_enter(regs);
+
local_daif_inherit(regs);
do_el1_fpac(regs, esr);
local_daif_mask();
- exit_to_kernel_mode(regs);
+ irqentry_exit(regs, state);
}
asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
@@ -522,23 +298,22 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
static __always_inline void __el1_pnmi(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
- arm64_enter_nmi(regs);
+ irqentry_state_t state = irqentry_nmi_enter(regs);
+
do_interrupt_handler(regs, handler);
- arm64_exit_nmi(regs);
+ irqentry_nmi_exit(regs, state);
}
static __always_inline void __el1_irq(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
- enter_from_kernel_mode(regs);
+ irqentry_state_t state = irqentry_enter(regs);
irq_enter_rcu();
do_interrupt_handler(regs, handler);
irq_exit_rcu();
- arm64_preempt_schedule_irq();
-
- exit_to_kernel_mode(regs);
+ irqentry_exit(regs, state);
}
static void noinstr el1_interrupt(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
@@ -564,21 +339,22 @@ asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
+ irqentry_state_t state;
local_daif_restore(DAIF_ERRCTX);
- arm64_enter_nmi(regs);
+ state = irqentry_nmi_enter(regs);
do_serror(regs, esr);
- arm64_exit_nmi(regs);
+ irqentry_nmi_exit(regs, state);
}
static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
{
unsigned long far = read_sysreg(far_el1);
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_mem_abort(far, esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
@@ -593,50 +369,50 @@ static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
if (!is_ttbr0_addr(far))
arm64_apply_bp_hardening();
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_mem_abort(far, esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_fpsimd_acc(esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_sve_acc(esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_sme_acc(esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_fpsimd_exc(esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_el0_sys(esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
@@ -646,50 +422,50 @@ static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
if (!is_ttbr0_addr(instruction_pointer(regs)))
arm64_apply_bp_hardening();
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_sp_pc_abort(far, esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_sp_pc_abort(regs->sp, esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_el0_undef(regs, esr);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_bti(struct pt_regs *regs)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_el0_bti(regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_el0_mops(regs, esr);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
bad_el0_sync(regs, 0, esr);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
@@ -697,28 +473,28 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
unsigned long far = read_sysreg(far_el1);
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
do_debug_exception(far, esr, regs);
local_daif_restore(DAIF_PROCCTX);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_svc(struct pt_regs *regs)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
cortex_a76_erratum_1463225_svc_handler();
fp_user_discard();
local_daif_restore(DAIF_PROCCTX);
do_el0_svc(regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_el0_fpac(regs, esr);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
@@ -783,7 +559,7 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
static void noinstr el0_interrupt(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
@@ -794,7 +570,7 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
do_interrupt_handler(regs, handler);
irq_exit_rcu();
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
@@ -820,14 +596,15 @@ asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
static void noinstr __el0_error_handler_common(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
+ irqentry_state_t state_nmi;
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_ERRCTX);
- arm64_enter_nmi(regs);
+ state_nmi = irqentry_nmi_enter(regs);
do_serror(regs, esr);
- arm64_exit_nmi(regs);
+ irqentry_nmi_exit(regs, state_nmi);
local_daif_restore(DAIF_PROCCTX);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
@@ -838,19 +615,19 @@ asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
#ifdef CONFIG_COMPAT
static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
do_el0_cp15(esr, regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
static void noinstr el0_svc_compat(struct pt_regs *regs)
{
- enter_from_user_mode(regs);
+ irqentry_enter_from_user_mode(regs);
cortex_a76_erratum_1463225_svc_handler();
local_daif_restore(DAIF_PROCCTX);
do_el0_svc_compat(regs);
- exit_to_user_mode(regs);
+ exit_to_user_mode_wrapper(regs);
}
asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
@@ -924,7 +701,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
unsigned long esr = read_sysreg(esr_el1);
unsigned long far = read_sysreg(far_el1);
- arm64_enter_nmi(regs);
+ irqentry_nmi_enter(regs);
panic_bad_stack(regs, esr, far);
}
#endif /* CONFIG_VMAP_STACK */
@@ -933,6 +710,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
asmlinkage noinstr unsigned long
__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
{
+ irqentry_state_t state;
unsigned long ret;
/*
@@ -957,9 +735,9 @@ __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
else if (cpu_has_pan())
set_pstate_pan(0);
- arm64_enter_nmi(regs);
+ state = irqentry_nmi_enter(regs);
ret = do_sdei_event(regs, arg);
- arm64_exit_nmi(regs);
+ irqentry_nmi_exit(regs, state);
return ret;
}
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 60fd85d5119d..9cc0f450fa76 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -41,9 +41,6 @@
#include <asm/traps.h>
#include <asm/system_misc.h>
-#define CREATE_TRACE_POINTS
-#include <trace/events/syscalls.h>
-
struct pt_regs_offset {
const char *name;
int offset;
@@ -2179,104 +2176,6 @@ long arch_ptrace(struct task_struct *child, long request,
return ptrace_request(child, request, addr, data);
}
-enum ptrace_syscall_dir {
- PTRACE_SYSCALL_ENTER = 0,
- PTRACE_SYSCALL_EXIT,
-};
-
-static void report_syscall_enter(struct pt_regs *regs)
-{
- int regno;
- unsigned long saved_reg;
-
- /*
- * We have some ABI weirdness here in the way that we handle syscall
- * exit stops because we indicate whether or not the stop has been
- * signalled from syscall entry or syscall exit by clobbering a general
- * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
- * and restoring its old value after the stop. This means that:
- *
- * - Any writes by the tracer to this register during the stop are
- * ignored/discarded.
- *
- * - The actual value of the register is not available during the stop,
- * so the tracer cannot save it and restore it later.
- *
- * - Syscall stops behave differently to seccomp and pseudo-step traps
- * (the latter do not nobble any registers).
- */
- regno = (is_compat_task() ? 12 : 7);
- saved_reg = regs->regs[regno];
- regs->regs[regno] = PTRACE_SYSCALL_ENTER;
-
- if (ptrace_report_syscall_entry(regs))
- forget_syscall(regs);
- regs->regs[regno] = saved_reg;
-}
-
-static void report_syscall_exit(struct pt_regs *regs)
-{
- int regno;
- unsigned long saved_reg;
-
- /* See comment for report_syscall_enter() */
- regno = (is_compat_task() ? 12 : 7);
- saved_reg = regs->regs[regno];
- regs->regs[regno] = PTRACE_SYSCALL_EXIT;
-
- if (!test_thread_flag(TIF_SINGLESTEP)) {
- ptrace_report_syscall_exit(regs, 0);
- regs->regs[regno] = saved_reg;
- } else {
- regs->regs[regno] = saved_reg;
-
- /*
- * Signal a pseudo-step exception since we are stepping but
- * tracer modifications to the registers may have rewound the
- * state machine.
- */
- ptrace_report_syscall_exit(regs, 1);
- }
-}
-
-int syscall_trace_enter(struct pt_regs *regs)
-{
- unsigned long flags = read_thread_flags();
-
- if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
- report_syscall_enter(regs);
- if (flags & _TIF_SYSCALL_EMU)
- return NO_SYSCALL;
- }
-
- /* Do the secure computing after ptrace; failures should be fast. */
- if (secure_computing() == -1)
- return NO_SYSCALL;
-
- if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
- trace_sys_enter(regs, regs->syscallno);
-
- audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
- regs->regs[2], regs->regs[3]);
-
- return regs->syscallno;
-}
-
-void syscall_trace_exit(struct pt_regs *regs)
-{
- unsigned long flags = read_thread_flags();
-
- audit_syscall_exit(regs);
-
- if (flags & _TIF_SYSCALL_TRACEPOINT)
- trace_sys_exit(regs, syscall_get_return_value(current, regs));
-
- if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP))
- report_syscall_exit(regs);
-
- rseq_syscall(regs);
-}
-
/*
* SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
* We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 4a77f4976e11..2982f6db6d96 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -19,6 +19,7 @@
#include <linux/ratelimit.h>
#include <linux/rseq.h>
#include <linux/syscalls.h>
+#include <linux/entry-common.h>
#include <asm/daifflags.h>
#include <asm/debug-monitors.h>
@@ -1266,7 +1267,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
* the kernel can handle, and then we build all the user-level signal handling
* stack-frames in one go after that.
*/
-void do_signal(struct pt_regs *regs)
+void arch_do_signal_or_restart(struct pt_regs *regs)
{
unsigned long continue_addr = 0, restart_addr = 0;
int retval = 0;
diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c
index ad198262b981..160ac9d15c27 100644
--- a/arch/arm64/kernel/syscall.c
+++ b/arch/arm64/kernel/syscall.c
@@ -7,6 +7,7 @@
#include <linux/ptrace.h>
#include <linux/randomize_kstack.h>
#include <linux/syscalls.h>
+#include <linux/entry-common.h>
#include <asm/debug-monitors.h>
#include <asm/exception.h>
@@ -66,14 +67,15 @@ static void invoke_syscall(struct pt_regs *regs, unsigned int scno,
choose_random_kstack_offset(get_random_u16() & 0x1FF);
}
-static inline bool has_syscall_work(unsigned long flags)
+static inline bool has_syscall_work(unsigned long work)
{
- return unlikely(flags & _TIF_SYSCALL_WORK);
+ return unlikely(work & SYSCALL_WORK_ENTER);
}
static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
const syscall_fn_t syscall_table[])
{
+ unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
unsigned long flags = read_thread_flags();
regs->orig_x0 = regs->regs[0];
@@ -107,7 +109,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
return;
}
- if (has_syscall_work(flags)) {
+ if (has_syscall_work(work)) {
/*
* The de-facto standard way to skip a system call using ptrace
* is to set the system call to -1 (NO_SYSCALL) and set x0 to a
@@ -125,7 +127,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
*/
if (scno == NO_SYSCALL)
syscall_set_return_value(current, regs, -ENOSYS, 0);
- scno = syscall_trace_enter(regs);
+ scno = syscall_trace_enter(regs, regs->syscallno, work);
if (scno == NO_SYSCALL)
goto trace_exit;
}
@@ -137,14 +139,14 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
* check again. However, if we were tracing entry, then we always trace
* exit regardless, as the old entry assembly did.
*/
- if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
- flags = read_thread_flags();
- if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
+ if (!has_syscall_work(work) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
+ work = READ_ONCE(current_thread_info()->syscall_work);
+ if (!has_syscall_work(work) && !report_single_step(work))
return;
}
trace_exit:
- syscall_trace_exit(regs);
+ syscall_exit_work(regs, work);
}
void do_el0_svc(struct pt_regs *regs)
--
2.34.1Hi, Catalin, Will and Mark, do you have any good suggestions and ideas?
On 2024/6/29 16:56, Jinjie Ruan wrote:
> Currently, x86, Riscv, Loongarch use the generic entry. Convert arm64
> to use the generic entry infrastructure from kernel/entry/*. The generic
> entry makes maintainers' work easier and codes more elegant, which also
> removed duplicate 150+ LOC. The changes are below:
>
> - Remove TIF_SYSCALL_* flag, _TIF_WORK_MASK, _TIF_SYSCALL_WORK
>
> - Remove syscall_trace_enter/exit() and use generic one.
>
> - Remove *enter_from/exit_to_kernel_mode(), and wrap with generic
> irqentry_enter/exit().
>
> - Remove *enter_from/exit_to_user_mode(), and wrap with generic
> irqentry_enter_from/exit_to_user_mode().
>
> - Remove arm64_enter/exit_nmi() and use generic irqentry_nmi_enter/exit().
>
> - Remove PREEMPT_DYNAMIC code, as generic entry will do it ok by
> implementing arch_irqentry_exit_need_resched().
>
> Tested ok with following test cases on Qemu cortex-a53 and HiSilicon
> Kunpeng-920:
>
> - Run `perf top` command
>
> - Switch between different `dynamic preempt` mode
>
> - Use `pseudo nmi`
>
> - stress-ng CPU stress test.
>
> - MTE test case in Documentation/arch/arm64/memory-tagging-extension.rst
> and all test cases in tools/testing/selftests/arm64/mte/* (Only Qemu).
>
> Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com>
> Tested-by: Jinjie Ruan <ruanjinjie@huawei.com>
> Tested-by: Kees Cook <kees@kernel.org>
> ---
> v3:
> - Test the MTE test cases.
> - Handle forget_syscall() in arch_post_report_syscall_entry()
> - Rebased on the previous patch, so move the arch funcs to entry-common.h.
> - Change __always_inline() to inline for arch_enter_from_kernel_mode() and
> arch_exit_to_kernel_mode_prepare(), add inline for the other arch funcs.
> - Remove unused signal.h for entry-common.h.
>
> v2:
> - Add Tested-by.
> - Rebased on the refactored report_syscall() code.
> - Add comment for arch_prepare_report_syscall_exit().
> - Update the commit message.
> - Adjust entry-common.h header file inclusion to alphabetical order.
> ---
> arch/arm64/Kconfig | 1 +
> arch/arm64/include/asm/entry-common.h | 172 ++++++++++++
> arch/arm64/include/asm/syscall.h | 6 +-
> arch/arm64/include/asm/thread_info.h | 23 +-
> arch/arm64/kernel/entry-common.c | 368 +++++---------------------
> arch/arm64/kernel/ptrace.c | 101 -------
> arch/arm64/kernel/signal.c | 3 +-
> arch/arm64/kernel/syscall.c | 18 +-
> 8 files changed, 264 insertions(+), 428 deletions(-)
> create mode 100644 arch/arm64/include/asm/entry-common.h
>
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 5d91259ee7b5..e6ccc5ea06fe 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -138,6 +138,7 @@ config ARM64
> select GENERIC_CPU_DEVICES
> select GENERIC_CPU_VULNERABILITIES
> select GENERIC_EARLY_IOREMAP
> + select GENERIC_ENTRY
> select GENERIC_IDLE_POLL_SETUP
> select GENERIC_IOREMAP
> select GENERIC_IRQ_IPI
> diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h
> new file mode 100644
> index 000000000000..a14d62b2eb69
> --- /dev/null
> +++ b/arch/arm64/include/asm/entry-common.h
> @@ -0,0 +1,172 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +
> +#ifndef _ASM_ARM64_ENTRY_COMMON_H
> +#define _ASM_ARM64_ENTRY_COMMON_H
> +
> +#include <linux/thread_info.h>
> +
> +#include <asm/daifflags.h>
> +#include <asm/fpsimd.h>
> +#include <asm/mte.h>
> +#include <asm/stacktrace.h>
> +
> +enum ptrace_syscall_dir {
> + PTRACE_SYSCALL_ENTER = 0,
> + PTRACE_SYSCALL_EXIT,
> +};
> +
> +#define ARCH_EXIT_TO_USER_MODE_WORK (_TIF_MTE_ASYNC_FAULT | _TIF_FOREIGN_FPSTATE)
> +
> +static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs)
> +{
> + mte_disable_tco_entry(current);
> +}
> +
> +#define arch_enter_from_user_mode arch_enter_from_user_mode
> +
> +static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
> + unsigned long ti_work)
> +{
> + if (ti_work & _TIF_MTE_ASYNC_FAULT) {
> + clear_thread_flag(TIF_MTE_ASYNC_FAULT);
> + send_sig_fault(SIGSEGV, SEGV_MTEAERR, (void __user *)NULL, current);
> + }
> +
> + if (ti_work & _TIF_FOREIGN_FPSTATE)
> + fpsimd_restore_current_state();
> +}
> +
> +#define arch_exit_to_user_mode_work arch_exit_to_user_mode_work
> +
> +static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
> + unsigned long ti_work)
> +{
> + local_daif_mask();
> + mte_check_tfsr_exit();
> +}
> +
> +#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
> +
> +static inline void arch_enter_from_kernel_mode(struct pt_regs *regs)
> +{
> + mte_check_tfsr_entry();
> + mte_disable_tco_entry(current);
> +}
> +
> +#define arch_enter_from_kernel_mode arch_enter_from_kernel_mode
> +
> +static inline void arch_exit_to_kernel_mode_prepare(struct pt_regs *regs)
> +{
> + mte_check_tfsr_exit();
> +}
> +
> +#define arch_exit_to_kernel_mode_prepare arch_exit_to_kernel_mode_prepare
> +
> +static inline bool arch_irqentry_exit_need_resched(void)
> +{
> + /*
> + * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
> + * priority masking is used the GIC irqchip driver will clear DAIF.IF
> + * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
> + * DAIF we must have handled an NMI, so skip preemption.
> + */
> + if (system_uses_irq_prio_masking() && read_sysreg(daif))
> + return false;
> +
> + /*
> + * Preempting a task from an IRQ means we leave copies of PSTATE
> + * on the stack. cpufeature's enable calls may modify PSTATE, but
> + * resuming one of these preempted tasks would undo those changes.
> + *
> + * Only allow a task to be preempted once cpufeatures have been
> + * enabled.
> + */
> + if (!system_capabilities_finalized())
> + return false;
> +
> + return true;
> +}
> +
> +#define arch_irqentry_exit_need_resched arch_irqentry_exit_need_resched
> +
> +static inline unsigned long arch_prepare_report_syscall_entry(struct pt_regs *regs)
> +{
> + unsigned long saved_reg;
> + int regno;
> +
> + /*
> + * We have some ABI weirdness here in the way that we handle syscall
> + * exit stops because we indicate whether or not the stop has been
> + * signalled from syscall entry or syscall exit by clobbering a general
> + * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
> + * and restoring its old value after the stop. This means that:
> + *
> + * - Any writes by the tracer to this register during the stop are
> + * ignored/discarded.
> + *
> + * - The actual value of the register is not available during the stop,
> + * so the tracer cannot save it and restore it later.
> + *
> + * - Syscall stops behave differently to seccomp and pseudo-step traps
> + * (the latter do not nobble any registers).
> + */
> + regno = (is_compat_task() ? 12 : 7);
> + saved_reg = regs->regs[regno];
> + regs->regs[regno] = PTRACE_SYSCALL_ENTER;
> +
> + return saved_reg;
> +}
> +
> +#define arch_prepare_report_syscall_entry arch_prepare_report_syscall_entry
> +
> +static inline void arch_post_report_syscall_entry(struct pt_regs *regs,
> + unsigned long saved_reg, long ret)
> +{
> + int regno = (is_compat_task() ? 12 : 7);
> +
> + if (ret)
> + forget_syscall(regs);
> +
> + regs->regs[regno] = saved_reg;
> +}
> +
> +#define arch_post_report_syscall_entry arch_post_report_syscall_entry
> +
> +static inline unsigned long arch_prepare_report_syscall_exit(struct pt_regs *regs,
> + unsigned long work)
> +{
> + unsigned long saved_reg;
> + int regno;
> +
> + /* See comment for arch_prepare_report_syscall_entry() */
> + regno = (is_compat_task() ? 12 : 7);
> + saved_reg = regs->regs[regno];
> + regs->regs[regno] = PTRACE_SYSCALL_EXIT;
> +
> + if (report_single_step(work)) {
> + /*
> + * Signal a pseudo-step exception since we are stepping but
> + * tracer modifications to the registers may have rewound the
> + * state machine.
> + */
> + regs->regs[regno] = saved_reg;
> + }
> +
> + return saved_reg;
> +}
> +
> +#define arch_prepare_report_syscall_exit arch_prepare_report_syscall_exit
> +
> +static inline void arch_post_report_syscall_exit(struct pt_regs *regs,
> + unsigned long saved_reg,
> + unsigned long work)
> +{
> + int regno = (is_compat_task() ? 12 : 7);
> +
> + if (!report_single_step(work))
> + regs->regs[regno] = saved_reg;
> +}
> +
> +#define arch_post_report_syscall_exit arch_post_report_syscall_exit
> +
> +#endif /* _ASM_ARM64_ENTRY_COMMON_H */
> diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h
> index ab8e14b96f68..9891b15da4c3 100644
> --- a/arch/arm64/include/asm/syscall.h
> +++ b/arch/arm64/include/asm/syscall.h
> @@ -85,7 +85,9 @@ static inline int syscall_get_arch(struct task_struct *task)
> return AUDIT_ARCH_AARCH64;
> }
>
> -int syscall_trace_enter(struct pt_regs *regs);
> -void syscall_trace_exit(struct pt_regs *regs);
> +static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
> +{
> + return false;
> +}
>
> #endif /* __ASM_SYSCALL_H */
> diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
> index e72a3bf9e563..ec5d74c53bf9 100644
> --- a/arch/arm64/include/asm/thread_info.h
> +++ b/arch/arm64/include/asm/thread_info.h
> @@ -43,6 +43,7 @@ struct thread_info {
> void *scs_sp;
> #endif
> u32 cpu;
> + unsigned long syscall_work; /* SYSCALL_WORK_ flags */
> };
>
> #define thread_saved_pc(tsk) \
> @@ -64,11 +65,6 @@ void arch_setup_new_exec(void);
> #define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */
> #define TIF_MTE_ASYNC_FAULT 5 /* MTE Asynchronous Tag Check Fault */
> #define TIF_NOTIFY_SIGNAL 6 /* signal notifications exist */
> -#define TIF_SYSCALL_TRACE 8 /* syscall trace active */
> -#define TIF_SYSCALL_AUDIT 9 /* syscall auditing */
> -#define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */
> -#define TIF_SECCOMP 11 /* syscall secure computing */
> -#define TIF_SYSCALL_EMU 12 /* syscall emulation active */
> #define TIF_MEMDIE 18 /* is terminating due to OOM killer */
> #define TIF_FREEZE 19
> #define TIF_RESTORE_SIGMASK 20
> @@ -86,27 +82,12 @@ void arch_setup_new_exec(void);
> #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
> #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
> #define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)
> -#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
> -#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
> -#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
> -#define _TIF_SECCOMP (1 << TIF_SECCOMP)
> -#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
> -#define _TIF_UPROBE (1 << TIF_UPROBE)
> -#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
> +#define _TIF_UPROBE (1 << TIF_UPROBE)
> #define _TIF_32BIT (1 << TIF_32BIT)
> #define _TIF_SVE (1 << TIF_SVE)
> #define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT)
> #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
>
> -#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
> - _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
> - _TIF_UPROBE | _TIF_MTE_ASYNC_FAULT | \
> - _TIF_NOTIFY_SIGNAL)
> -
> -#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
> - _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
> - _TIF_SYSCALL_EMU)
> -
> #ifdef CONFIG_SHADOW_CALL_STACK
> #define INIT_SCS \
> .scs_base = init_shadow_call_stack, \
> diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
> index b77a15955f28..ac61aeb93933 100644
> --- a/arch/arm64/kernel/entry-common.c
> +++ b/arch/arm64/kernel/entry-common.c
> @@ -14,6 +14,7 @@
> #include <linux/sched.h>
> #include <linux/sched/debug.h>
> #include <linux/thread_info.h>
> +#include <linux/entry-common.h>
>
> #include <asm/cpufeature.h>
> #include <asm/daifflags.h>
> @@ -28,201 +29,15 @@
> #include <asm/sysreg.h>
> #include <asm/system_misc.h>
>
> -/*
> - * Handle IRQ/context state management when entering from kernel mode.
> - * Before this function is called it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - *
> - * This is intended to match the logic in irqentry_enter(), handling the kernel
> - * mode transitions only.
> - */
> -static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)
> -{
> - regs->exit_rcu = false;
> -
> - if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
> - lockdep_hardirqs_off(CALLER_ADDR0);
> - ct_irq_enter();
> - trace_hardirqs_off_finish();
> -
> - regs->exit_rcu = true;
> - return;
> - }
> -
> - lockdep_hardirqs_off(CALLER_ADDR0);
> - rcu_irq_enter_check_tick();
> - trace_hardirqs_off_finish();
> -}
> -
> -static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
> -{
> - __enter_from_kernel_mode(regs);
> - mte_check_tfsr_entry();
> - mte_disable_tco_entry(current);
> -}
> -
> -/*
> - * Handle IRQ/context state management when exiting to kernel mode.
> - * After this function returns it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - *
> - * This is intended to match the logic in irqentry_exit(), handling the kernel
> - * mode transitions only, and with preemption handled elsewhere.
> - */
> -static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)
> -{
> - lockdep_assert_irqs_disabled();
> -
> - if (interrupts_enabled(regs)) {
> - if (regs->exit_rcu) {
> - trace_hardirqs_on_prepare();
> - lockdep_hardirqs_on_prepare();
> - ct_irq_exit();
> - lockdep_hardirqs_on(CALLER_ADDR0);
> - return;
> - }
> -
> - trace_hardirqs_on();
> - } else {
> - if (regs->exit_rcu)
> - ct_irq_exit();
> - }
> -}
> -
> -static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
> -{
> - mte_check_tfsr_exit();
> - __exit_to_kernel_mode(regs);
> -}
> -
> -/*
> - * Handle IRQ/context state management when entering from user mode.
> - * Before this function is called it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - */
> -static __always_inline void __enter_from_user_mode(void)
> +static __always_inline void exit_to_user_mode_wrapper(struct pt_regs *regs)
> {
> - lockdep_hardirqs_off(CALLER_ADDR0);
> - CT_WARN_ON(ct_state() != CONTEXT_USER);
> - user_exit_irqoff();
> - trace_hardirqs_off_finish();
> - mte_disable_tco_entry(current);
> -}
> -
> -static __always_inline void enter_from_user_mode(struct pt_regs *regs)
> -{
> - __enter_from_user_mode();
> -}
> -
> -/*
> - * Handle IRQ/context state management when exiting to user mode.
> - * After this function returns it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - */
> -static __always_inline void __exit_to_user_mode(void)
> -{
> - trace_hardirqs_on_prepare();
> - lockdep_hardirqs_on_prepare();
> - user_enter_irqoff();
> - lockdep_hardirqs_on(CALLER_ADDR0);
> -}
> -
> -static void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags)
> -{
> - do {
> - local_irq_enable();
> -
> - if (thread_flags & _TIF_NEED_RESCHED)
> - schedule();
> -
> - if (thread_flags & _TIF_UPROBE)
> - uprobe_notify_resume(regs);
> -
> - if (thread_flags & _TIF_MTE_ASYNC_FAULT) {
> - clear_thread_flag(TIF_MTE_ASYNC_FAULT);
> - send_sig_fault(SIGSEGV, SEGV_MTEAERR,
> - (void __user *)NULL, current);
> - }
> -
> - if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
> - do_signal(regs);
> -
> - if (thread_flags & _TIF_NOTIFY_RESUME)
> - resume_user_mode_work(regs);
> -
> - if (thread_flags & _TIF_FOREIGN_FPSTATE)
> - fpsimd_restore_current_state();
> -
> - local_irq_disable();
> - thread_flags = read_thread_flags();
> - } while (thread_flags & _TIF_WORK_MASK);
> -}
> -
> -static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)
> -{
> - unsigned long flags;
> -
> local_irq_disable();
> -
> - flags = read_thread_flags();
> - if (unlikely(flags & _TIF_WORK_MASK))
> - do_notify_resume(regs, flags);
> -
> - local_daif_mask();
> -
> - lockdep_sys_exit();
> -}
> -
> -static __always_inline void exit_to_user_mode(struct pt_regs *regs)
> -{
> - exit_to_user_mode_prepare(regs);
> - mte_check_tfsr_exit();
> - __exit_to_user_mode();
> + irqentry_exit_to_user_mode(regs);
> }
>
> asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
> {
> - exit_to_user_mode(regs);
> -}
> -
> -/*
> - * Handle IRQ/context state management when entering an NMI from user/kernel
> - * mode. Before this function is called it is not safe to call regular kernel
> - * code, instrumentable code, or any code which may trigger an exception.
> - */
> -static void noinstr arm64_enter_nmi(struct pt_regs *regs)
> -{
> - regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
> -
> - __nmi_enter();
> - lockdep_hardirqs_off(CALLER_ADDR0);
> - lockdep_hardirq_enter();
> - ct_nmi_enter();
> -
> - trace_hardirqs_off_finish();
> - ftrace_nmi_enter();
> -}
> -
> -/*
> - * Handle IRQ/context state management when exiting an NMI from user/kernel
> - * mode. After this function returns it is not safe to call regular kernel
> - * code, instrumentable code, or any code which may trigger an exception.
> - */
> -static void noinstr arm64_exit_nmi(struct pt_regs *regs)
> -{
> - bool restore = regs->lockdep_hardirqs;
> -
> - ftrace_nmi_exit();
> - if (restore) {
> - trace_hardirqs_on_prepare();
> - lockdep_hardirqs_on_prepare();
> - }
> -
> - ct_nmi_exit();
> - lockdep_hardirq_exit();
> - if (restore)
> - lockdep_hardirqs_on(CALLER_ADDR0);
> - __nmi_exit();
> + exit_to_user_mode_wrapper(regs);
> }
>
> /*
> @@ -259,48 +74,6 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
> lockdep_hardirqs_on(CALLER_ADDR0);
> }
>
> -#ifdef CONFIG_PREEMPT_DYNAMIC
> -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
> -#define need_irq_preemption() \
> - (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
> -#else
> -#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION))
> -#endif
> -
> -static void __sched arm64_preempt_schedule_irq(void)
> -{
> - if (!need_irq_preemption())
> - return;
> -
> - /*
> - * Note: thread_info::preempt_count includes both thread_info::count
> - * and thread_info::need_resched, and is not equivalent to
> - * preempt_count().
> - */
> - if (READ_ONCE(current_thread_info()->preempt_count) != 0)
> - return;
> -
> - /*
> - * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
> - * priority masking is used the GIC irqchip driver will clear DAIF.IF
> - * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
> - * DAIF we must have handled an NMI, so skip preemption.
> - */
> - if (system_uses_irq_prio_masking() && read_sysreg(daif))
> - return;
> -
> - /*
> - * Preempting a task from an IRQ means we leave copies of PSTATE
> - * on the stack. cpufeature's enable calls may modify PSTATE, but
> - * resuming one of these preempted tasks would undo those changes.
> - *
> - * Only allow a task to be preempted once cpufeatures have been
> - * enabled.
> - */
> - if (system_capabilities_finalized())
> - preempt_schedule_irq();
> -}
> -
> static void do_interrupt_handler(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> {
> @@ -320,7 +93,7 @@ extern void (*handle_arch_fiq)(struct pt_regs *);
> static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
> unsigned long esr)
> {
> - arm64_enter_nmi(regs);
> + irqentry_nmi_enter(regs);
>
> console_verbose();
>
> @@ -426,41 +199,43 @@ UNHANDLED(el1t, 64, error)
> static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
> {
> unsigned long far = read_sysreg(far_el1);
> + irqentry_state_t state = irqentry_enter(regs);
>
> - enter_from_kernel_mode(regs);
> local_daif_inherit(regs);
> do_mem_abort(far, esr, regs);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
> {
> unsigned long far = read_sysreg(far_el1);
> + irqentry_state_t state = irqentry_enter(regs);
>
> - enter_from_kernel_mode(regs);
> local_daif_inherit(regs);
> do_sp_pc_abort(far, esr, regs);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_kernel_mode(regs);
> + irqentry_state_t state = irqentry_enter(regs);
> +
> local_daif_inherit(regs);
> do_el1_undef(regs, esr);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_kernel_mode(regs);
> + irqentry_state_t state = irqentry_enter(regs);
> +
> local_daif_inherit(regs);
> do_el1_bti(regs, esr);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
> @@ -475,11 +250,12 @@ static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
>
> static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_kernel_mode(regs);
> + irqentry_state_t state = irqentry_enter(regs);
> +
> local_daif_inherit(regs);
> do_el1_fpac(regs, esr);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
> @@ -522,23 +298,22 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
> static __always_inline void __el1_pnmi(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> {
> - arm64_enter_nmi(regs);
> + irqentry_state_t state = irqentry_nmi_enter(regs);
> +
> do_interrupt_handler(regs, handler);
> - arm64_exit_nmi(regs);
> + irqentry_nmi_exit(regs, state);
> }
>
> static __always_inline void __el1_irq(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> {
> - enter_from_kernel_mode(regs);
> + irqentry_state_t state = irqentry_enter(regs);
>
> irq_enter_rcu();
> do_interrupt_handler(regs, handler);
> irq_exit_rcu();
>
> - arm64_preempt_schedule_irq();
> -
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
> static void noinstr el1_interrupt(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> @@ -564,21 +339,22 @@ asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
> asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
> {
> unsigned long esr = read_sysreg(esr_el1);
> + irqentry_state_t state;
>
> local_daif_restore(DAIF_ERRCTX);
> - arm64_enter_nmi(regs);
> + state = irqentry_nmi_enter(regs);
> do_serror(regs, esr);
> - arm64_exit_nmi(regs);
> + irqentry_nmi_exit(regs, state);
> }
>
> static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
> {
> unsigned long far = read_sysreg(far_el1);
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_mem_abort(far, esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
> @@ -593,50 +369,50 @@ static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
> if (!is_ttbr0_addr(far))
> arm64_apply_bp_hardening();
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_mem_abort(far, esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_fpsimd_acc(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_sve_acc(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_sme_acc(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_fpsimd_exc(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_sys(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
> @@ -646,50 +422,50 @@ static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
> if (!is_ttbr0_addr(instruction_pointer(regs)))
> arm64_apply_bp_hardening();
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_sp_pc_abort(far, esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_sp_pc_abort(regs->sp, esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_undef(regs, esr);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_bti(struct pt_regs *regs)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_bti(regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_mops(regs, esr);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> bad_el0_sync(regs, 0, esr);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
> @@ -697,28 +473,28 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
> /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
> unsigned long far = read_sysreg(far_el1);
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> do_debug_exception(far, esr, regs);
> local_daif_restore(DAIF_PROCCTX);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_svc(struct pt_regs *regs)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> cortex_a76_erratum_1463225_svc_handler();
> fp_user_discard();
> local_daif_restore(DAIF_PROCCTX);
> do_el0_svc(regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_fpac(regs, esr);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
> @@ -783,7 +559,7 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
> static void noinstr el0_interrupt(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
>
> write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
>
> @@ -794,7 +570,7 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
> do_interrupt_handler(regs, handler);
> irq_exit_rcu();
>
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
> @@ -820,14 +596,15 @@ asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
> static void noinstr __el0_error_handler_common(struct pt_regs *regs)
> {
> unsigned long esr = read_sysreg(esr_el1);
> + irqentry_state_t state_nmi;
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_ERRCTX);
> - arm64_enter_nmi(regs);
> + state_nmi = irqentry_nmi_enter(regs);
> do_serror(regs, esr);
> - arm64_exit_nmi(regs);
> + irqentry_nmi_exit(regs, state_nmi);
> local_daif_restore(DAIF_PROCCTX);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
> @@ -838,19 +615,19 @@ asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
> #ifdef CONFIG_COMPAT
> static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_cp15(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_svc_compat(struct pt_regs *regs)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> cortex_a76_erratum_1463225_svc_handler();
> local_daif_restore(DAIF_PROCCTX);
> do_el0_svc_compat(regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
> @@ -924,7 +701,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
> unsigned long esr = read_sysreg(esr_el1);
> unsigned long far = read_sysreg(far_el1);
>
> - arm64_enter_nmi(regs);
> + irqentry_nmi_enter(regs);
> panic_bad_stack(regs, esr, far);
> }
> #endif /* CONFIG_VMAP_STACK */
> @@ -933,6 +710,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
> asmlinkage noinstr unsigned long
> __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
> {
> + irqentry_state_t state;
> unsigned long ret;
>
> /*
> @@ -957,9 +735,9 @@ __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
> else if (cpu_has_pan())
> set_pstate_pan(0);
>
> - arm64_enter_nmi(regs);
> + state = irqentry_nmi_enter(regs);
> ret = do_sdei_event(regs, arg);
> - arm64_exit_nmi(regs);
> + irqentry_nmi_exit(regs, state);
>
> return ret;
> }
> diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
> index 60fd85d5119d..9cc0f450fa76 100644
> --- a/arch/arm64/kernel/ptrace.c
> +++ b/arch/arm64/kernel/ptrace.c
> @@ -41,9 +41,6 @@
> #include <asm/traps.h>
> #include <asm/system_misc.h>
>
> -#define CREATE_TRACE_POINTS
> -#include <trace/events/syscalls.h>
> -
> struct pt_regs_offset {
> const char *name;
> int offset;
> @@ -2179,104 +2176,6 @@ long arch_ptrace(struct task_struct *child, long request,
> return ptrace_request(child, request, addr, data);
> }
>
> -enum ptrace_syscall_dir {
> - PTRACE_SYSCALL_ENTER = 0,
> - PTRACE_SYSCALL_EXIT,
> -};
> -
> -static void report_syscall_enter(struct pt_regs *regs)
> -{
> - int regno;
> - unsigned long saved_reg;
> -
> - /*
> - * We have some ABI weirdness here in the way that we handle syscall
> - * exit stops because we indicate whether or not the stop has been
> - * signalled from syscall entry or syscall exit by clobbering a general
> - * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
> - * and restoring its old value after the stop. This means that:
> - *
> - * - Any writes by the tracer to this register during the stop are
> - * ignored/discarded.
> - *
> - * - The actual value of the register is not available during the stop,
> - * so the tracer cannot save it and restore it later.
> - *
> - * - Syscall stops behave differently to seccomp and pseudo-step traps
> - * (the latter do not nobble any registers).
> - */
> - regno = (is_compat_task() ? 12 : 7);
> - saved_reg = regs->regs[regno];
> - regs->regs[regno] = PTRACE_SYSCALL_ENTER;
> -
> - if (ptrace_report_syscall_entry(regs))
> - forget_syscall(regs);
> - regs->regs[regno] = saved_reg;
> -}
> -
> -static void report_syscall_exit(struct pt_regs *regs)
> -{
> - int regno;
> - unsigned long saved_reg;
> -
> - /* See comment for report_syscall_enter() */
> - regno = (is_compat_task() ? 12 : 7);
> - saved_reg = regs->regs[regno];
> - regs->regs[regno] = PTRACE_SYSCALL_EXIT;
> -
> - if (!test_thread_flag(TIF_SINGLESTEP)) {
> - ptrace_report_syscall_exit(regs, 0);
> - regs->regs[regno] = saved_reg;
> - } else {
> - regs->regs[regno] = saved_reg;
> -
> - /*
> - * Signal a pseudo-step exception since we are stepping but
> - * tracer modifications to the registers may have rewound the
> - * state machine.
> - */
> - ptrace_report_syscall_exit(regs, 1);
> - }
> -}
> -
> -int syscall_trace_enter(struct pt_regs *regs)
> -{
> - unsigned long flags = read_thread_flags();
> -
> - if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
> - report_syscall_enter(regs);
> - if (flags & _TIF_SYSCALL_EMU)
> - return NO_SYSCALL;
> - }
> -
> - /* Do the secure computing after ptrace; failures should be fast. */
> - if (secure_computing() == -1)
> - return NO_SYSCALL;
> -
> - if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
> - trace_sys_enter(regs, regs->syscallno);
> -
> - audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
> - regs->regs[2], regs->regs[3]);
> -
> - return regs->syscallno;
> -}
> -
> -void syscall_trace_exit(struct pt_regs *regs)
> -{
> - unsigned long flags = read_thread_flags();
> -
> - audit_syscall_exit(regs);
> -
> - if (flags & _TIF_SYSCALL_TRACEPOINT)
> - trace_sys_exit(regs, syscall_get_return_value(current, regs));
> -
> - if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP))
> - report_syscall_exit(regs);
> -
> - rseq_syscall(regs);
> -}
> -
> /*
> * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
> * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
> diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
> index 4a77f4976e11..2982f6db6d96 100644
> --- a/arch/arm64/kernel/signal.c
> +++ b/arch/arm64/kernel/signal.c
> @@ -19,6 +19,7 @@
> #include <linux/ratelimit.h>
> #include <linux/rseq.h>
> #include <linux/syscalls.h>
> +#include <linux/entry-common.h>
>
> #include <asm/daifflags.h>
> #include <asm/debug-monitors.h>
> @@ -1266,7 +1267,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
> * the kernel can handle, and then we build all the user-level signal handling
> * stack-frames in one go after that.
> */
> -void do_signal(struct pt_regs *regs)
> +void arch_do_signal_or_restart(struct pt_regs *regs)
> {
> unsigned long continue_addr = 0, restart_addr = 0;
> int retval = 0;
> diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c
> index ad198262b981..160ac9d15c27 100644
> --- a/arch/arm64/kernel/syscall.c
> +++ b/arch/arm64/kernel/syscall.c
> @@ -7,6 +7,7 @@
> #include <linux/ptrace.h>
> #include <linux/randomize_kstack.h>
> #include <linux/syscalls.h>
> +#include <linux/entry-common.h>
>
> #include <asm/debug-monitors.h>
> #include <asm/exception.h>
> @@ -66,14 +67,15 @@ static void invoke_syscall(struct pt_regs *regs, unsigned int scno,
> choose_random_kstack_offset(get_random_u16() & 0x1FF);
> }
>
> -static inline bool has_syscall_work(unsigned long flags)
> +static inline bool has_syscall_work(unsigned long work)
> {
> - return unlikely(flags & _TIF_SYSCALL_WORK);
> + return unlikely(work & SYSCALL_WORK_ENTER);
> }
>
> static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> const syscall_fn_t syscall_table[])
> {
> + unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
> unsigned long flags = read_thread_flags();
>
> regs->orig_x0 = regs->regs[0];
> @@ -107,7 +109,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> return;
> }
>
> - if (has_syscall_work(flags)) {
> + if (has_syscall_work(work)) {
> /*
> * The de-facto standard way to skip a system call using ptrace
> * is to set the system call to -1 (NO_SYSCALL) and set x0 to a
> @@ -125,7 +127,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> */
> if (scno == NO_SYSCALL)
> syscall_set_return_value(current, regs, -ENOSYS, 0);
> - scno = syscall_trace_enter(regs);
> + scno = syscall_trace_enter(regs, regs->syscallno, work);
> if (scno == NO_SYSCALL)
> goto trace_exit;
> }
> @@ -137,14 +139,14 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> * check again. However, if we were tracing entry, then we always trace
> * exit regardless, as the old entry assembly did.
> */
> - if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
> - flags = read_thread_flags();
> - if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
> + if (!has_syscall_work(work) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
> + work = READ_ONCE(current_thread_info()->syscall_work);
> + if (!has_syscall_work(work) && !report_single_step(work))
> return;
> }
>
> trace_exit:
> - syscall_trace_exit(regs);
> + syscall_exit_work(regs, work);
> }
>
> void do_el0_svc(struct pt_regs *regs)
On Wed, Sep 18, 2024 at 09:59:19AM +0800, Jinjie Ruan wrote:
> Hi, Catalin, Will and Mark, do you have any good suggestions and ideas?
Hi,
Sorry, this is on my queue of things to review, but I haven't had the
chance to review this in detail yet. This needs a thorough review for
things like noinstr safety, and I intend to take a look at this after
Linux Plumbers Conference.
Mark.
>
> On 2024/6/29 16:56, Jinjie Ruan wrote:
> > Currently, x86, Riscv, Loongarch use the generic entry. Convert arm64
> > to use the generic entry infrastructure from kernel/entry/*. The generic
> > entry makes maintainers' work easier and codes more elegant, which also
> > removed duplicate 150+ LOC. The changes are below:
> >
> > - Remove TIF_SYSCALL_* flag, _TIF_WORK_MASK, _TIF_SYSCALL_WORK
> >
> > - Remove syscall_trace_enter/exit() and use generic one.
> >
> > - Remove *enter_from/exit_to_kernel_mode(), and wrap with generic
> > irqentry_enter/exit().
> >
> > - Remove *enter_from/exit_to_user_mode(), and wrap with generic
> > irqentry_enter_from/exit_to_user_mode().
> >
> > - Remove arm64_enter/exit_nmi() and use generic irqentry_nmi_enter/exit().
> >
> > - Remove PREEMPT_DYNAMIC code, as generic entry will do it ok by
> > implementing arch_irqentry_exit_need_resched().
> >
> > Tested ok with following test cases on Qemu cortex-a53 and HiSilicon
> > Kunpeng-920:
> >
> > - Run `perf top` command
> >
> > - Switch between different `dynamic preempt` mode
> >
> > - Use `pseudo nmi`
> >
> > - stress-ng CPU stress test.
> >
> > - MTE test case in Documentation/arch/arm64/memory-tagging-extension.rst
> > and all test cases in tools/testing/selftests/arm64/mte/* (Only Qemu).
> >
> > Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com>
> > Tested-by: Jinjie Ruan <ruanjinjie@huawei.com>
> > Tested-by: Kees Cook <kees@kernel.org>
> > ---
> > v3:
> > - Test the MTE test cases.
> > - Handle forget_syscall() in arch_post_report_syscall_entry()
> > - Rebased on the previous patch, so move the arch funcs to entry-common.h.
> > - Change __always_inline() to inline for arch_enter_from_kernel_mode() and
> > arch_exit_to_kernel_mode_prepare(), add inline for the other arch funcs.
> > - Remove unused signal.h for entry-common.h.
> >
> > v2:
> > - Add Tested-by.
> > - Rebased on the refactored report_syscall() code.
> > - Add comment for arch_prepare_report_syscall_exit().
> > - Update the commit message.
> > - Adjust entry-common.h header file inclusion to alphabetical order.
> > ---
> > arch/arm64/Kconfig | 1 +
> > arch/arm64/include/asm/entry-common.h | 172 ++++++++++++
> > arch/arm64/include/asm/syscall.h | 6 +-
> > arch/arm64/include/asm/thread_info.h | 23 +-
> > arch/arm64/kernel/entry-common.c | 368 +++++---------------------
> > arch/arm64/kernel/ptrace.c | 101 -------
> > arch/arm64/kernel/signal.c | 3 +-
> > arch/arm64/kernel/syscall.c | 18 +-
> > 8 files changed, 264 insertions(+), 428 deletions(-)
> > create mode 100644 arch/arm64/include/asm/entry-common.h
> >
> > diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> > index 5d91259ee7b5..e6ccc5ea06fe 100644
> > --- a/arch/arm64/Kconfig
> > +++ b/arch/arm64/Kconfig
> > @@ -138,6 +138,7 @@ config ARM64
> > select GENERIC_CPU_DEVICES
> > select GENERIC_CPU_VULNERABILITIES
> > select GENERIC_EARLY_IOREMAP
> > + select GENERIC_ENTRY
> > select GENERIC_IDLE_POLL_SETUP
> > select GENERIC_IOREMAP
> > select GENERIC_IRQ_IPI
> > diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h
> > new file mode 100644
> > index 000000000000..a14d62b2eb69
> > --- /dev/null
> > +++ b/arch/arm64/include/asm/entry-common.h
> > @@ -0,0 +1,172 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +
> > +#ifndef _ASM_ARM64_ENTRY_COMMON_H
> > +#define _ASM_ARM64_ENTRY_COMMON_H
> > +
> > +#include <linux/thread_info.h>
> > +
> > +#include <asm/daifflags.h>
> > +#include <asm/fpsimd.h>
> > +#include <asm/mte.h>
> > +#include <asm/stacktrace.h>
> > +
> > +enum ptrace_syscall_dir {
> > + PTRACE_SYSCALL_ENTER = 0,
> > + PTRACE_SYSCALL_EXIT,
> > +};
> > +
> > +#define ARCH_EXIT_TO_USER_MODE_WORK (_TIF_MTE_ASYNC_FAULT | _TIF_FOREIGN_FPSTATE)
> > +
> > +static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs)
> > +{
> > + mte_disable_tco_entry(current);
> > +}
> > +
> > +#define arch_enter_from_user_mode arch_enter_from_user_mode
> > +
> > +static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
> > + unsigned long ti_work)
> > +{
> > + if (ti_work & _TIF_MTE_ASYNC_FAULT) {
> > + clear_thread_flag(TIF_MTE_ASYNC_FAULT);
> > + send_sig_fault(SIGSEGV, SEGV_MTEAERR, (void __user *)NULL, current);
> > + }
> > +
> > + if (ti_work & _TIF_FOREIGN_FPSTATE)
> > + fpsimd_restore_current_state();
> > +}
> > +
> > +#define arch_exit_to_user_mode_work arch_exit_to_user_mode_work
> > +
> > +static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
> > + unsigned long ti_work)
> > +{
> > + local_daif_mask();
> > + mte_check_tfsr_exit();
> > +}
> > +
> > +#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
> > +
> > +static inline void arch_enter_from_kernel_mode(struct pt_regs *regs)
> > +{
> > + mte_check_tfsr_entry();
> > + mte_disable_tco_entry(current);
> > +}
> > +
> > +#define arch_enter_from_kernel_mode arch_enter_from_kernel_mode
> > +
> > +static inline void arch_exit_to_kernel_mode_prepare(struct pt_regs *regs)
> > +{
> > + mte_check_tfsr_exit();
> > +}
> > +
> > +#define arch_exit_to_kernel_mode_prepare arch_exit_to_kernel_mode_prepare
> > +
> > +static inline bool arch_irqentry_exit_need_resched(void)
> > +{
> > + /*
> > + * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
> > + * priority masking is used the GIC irqchip driver will clear DAIF.IF
> > + * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
> > + * DAIF we must have handled an NMI, so skip preemption.
> > + */
> > + if (system_uses_irq_prio_masking() && read_sysreg(daif))
> > + return false;
> > +
> > + /*
> > + * Preempting a task from an IRQ means we leave copies of PSTATE
> > + * on the stack. cpufeature's enable calls may modify PSTATE, but
> > + * resuming one of these preempted tasks would undo those changes.
> > + *
> > + * Only allow a task to be preempted once cpufeatures have been
> > + * enabled.
> > + */
> > + if (!system_capabilities_finalized())
> > + return false;
> > +
> > + return true;
> > +}
> > +
> > +#define arch_irqentry_exit_need_resched arch_irqentry_exit_need_resched
> > +
> > +static inline unsigned long arch_prepare_report_syscall_entry(struct pt_regs *regs)
> > +{
> > + unsigned long saved_reg;
> > + int regno;
> > +
> > + /*
> > + * We have some ABI weirdness here in the way that we handle syscall
> > + * exit stops because we indicate whether or not the stop has been
> > + * signalled from syscall entry or syscall exit by clobbering a general
> > + * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
> > + * and restoring its old value after the stop. This means that:
> > + *
> > + * - Any writes by the tracer to this register during the stop are
> > + * ignored/discarded.
> > + *
> > + * - The actual value of the register is not available during the stop,
> > + * so the tracer cannot save it and restore it later.
> > + *
> > + * - Syscall stops behave differently to seccomp and pseudo-step traps
> > + * (the latter do not nobble any registers).
> > + */
> > + regno = (is_compat_task() ? 12 : 7);
> > + saved_reg = regs->regs[regno];
> > + regs->regs[regno] = PTRACE_SYSCALL_ENTER;
> > +
> > + return saved_reg;
> > +}
> > +
> > +#define arch_prepare_report_syscall_entry arch_prepare_report_syscall_entry
> > +
> > +static inline void arch_post_report_syscall_entry(struct pt_regs *regs,
> > + unsigned long saved_reg, long ret)
> > +{
> > + int regno = (is_compat_task() ? 12 : 7);
> > +
> > + if (ret)
> > + forget_syscall(regs);
> > +
> > + regs->regs[regno] = saved_reg;
> > +}
> > +
> > +#define arch_post_report_syscall_entry arch_post_report_syscall_entry
> > +
> > +static inline unsigned long arch_prepare_report_syscall_exit(struct pt_regs *regs,
> > + unsigned long work)
> > +{
> > + unsigned long saved_reg;
> > + int regno;
> > +
> > + /* See comment for arch_prepare_report_syscall_entry() */
> > + regno = (is_compat_task() ? 12 : 7);
> > + saved_reg = regs->regs[regno];
> > + regs->regs[regno] = PTRACE_SYSCALL_EXIT;
> > +
> > + if (report_single_step(work)) {
> > + /*
> > + * Signal a pseudo-step exception since we are stepping but
> > + * tracer modifications to the registers may have rewound the
> > + * state machine.
> > + */
> > + regs->regs[regno] = saved_reg;
> > + }
> > +
> > + return saved_reg;
> > +}
> > +
> > +#define arch_prepare_report_syscall_exit arch_prepare_report_syscall_exit
> > +
> > +static inline void arch_post_report_syscall_exit(struct pt_regs *regs,
> > + unsigned long saved_reg,
> > + unsigned long work)
> > +{
> > + int regno = (is_compat_task() ? 12 : 7);
> > +
> > + if (!report_single_step(work))
> > + regs->regs[regno] = saved_reg;
> > +}
> > +
> > +#define arch_post_report_syscall_exit arch_post_report_syscall_exit
> > +
> > +#endif /* _ASM_ARM64_ENTRY_COMMON_H */
> > diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h
> > index ab8e14b96f68..9891b15da4c3 100644
> > --- a/arch/arm64/include/asm/syscall.h
> > +++ b/arch/arm64/include/asm/syscall.h
> > @@ -85,7 +85,9 @@ static inline int syscall_get_arch(struct task_struct *task)
> > return AUDIT_ARCH_AARCH64;
> > }
> >
> > -int syscall_trace_enter(struct pt_regs *regs);
> > -void syscall_trace_exit(struct pt_regs *regs);
> > +static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
> > +{
> > + return false;
> > +}
> >
> > #endif /* __ASM_SYSCALL_H */
> > diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
> > index e72a3bf9e563..ec5d74c53bf9 100644
> > --- a/arch/arm64/include/asm/thread_info.h
> > +++ b/arch/arm64/include/asm/thread_info.h
> > @@ -43,6 +43,7 @@ struct thread_info {
> > void *scs_sp;
> > #endif
> > u32 cpu;
> > + unsigned long syscall_work; /* SYSCALL_WORK_ flags */
> > };
> >
> > #define thread_saved_pc(tsk) \
> > @@ -64,11 +65,6 @@ void arch_setup_new_exec(void);
> > #define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */
> > #define TIF_MTE_ASYNC_FAULT 5 /* MTE Asynchronous Tag Check Fault */
> > #define TIF_NOTIFY_SIGNAL 6 /* signal notifications exist */
> > -#define TIF_SYSCALL_TRACE 8 /* syscall trace active */
> > -#define TIF_SYSCALL_AUDIT 9 /* syscall auditing */
> > -#define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */
> > -#define TIF_SECCOMP 11 /* syscall secure computing */
> > -#define TIF_SYSCALL_EMU 12 /* syscall emulation active */
> > #define TIF_MEMDIE 18 /* is terminating due to OOM killer */
> > #define TIF_FREEZE 19
> > #define TIF_RESTORE_SIGMASK 20
> > @@ -86,27 +82,12 @@ void arch_setup_new_exec(void);
> > #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
> > #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
> > #define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)
> > -#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
> > -#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
> > -#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
> > -#define _TIF_SECCOMP (1 << TIF_SECCOMP)
> > -#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
> > -#define _TIF_UPROBE (1 << TIF_UPROBE)
> > -#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
> > +#define _TIF_UPROBE (1 << TIF_UPROBE)
> > #define _TIF_32BIT (1 << TIF_32BIT)
> > #define _TIF_SVE (1 << TIF_SVE)
> > #define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT)
> > #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
> >
> > -#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
> > - _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
> > - _TIF_UPROBE | _TIF_MTE_ASYNC_FAULT | \
> > - _TIF_NOTIFY_SIGNAL)
> > -
> > -#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
> > - _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
> > - _TIF_SYSCALL_EMU)
> > -
> > #ifdef CONFIG_SHADOW_CALL_STACK
> > #define INIT_SCS \
> > .scs_base = init_shadow_call_stack, \
> > diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
> > index b77a15955f28..ac61aeb93933 100644
> > --- a/arch/arm64/kernel/entry-common.c
> > +++ b/arch/arm64/kernel/entry-common.c
> > @@ -14,6 +14,7 @@
> > #include <linux/sched.h>
> > #include <linux/sched/debug.h>
> > #include <linux/thread_info.h>
> > +#include <linux/entry-common.h>
> >
> > #include <asm/cpufeature.h>
> > #include <asm/daifflags.h>
> > @@ -28,201 +29,15 @@
> > #include <asm/sysreg.h>
> > #include <asm/system_misc.h>
> >
> > -/*
> > - * Handle IRQ/context state management when entering from kernel mode.
> > - * Before this function is called it is not safe to call regular kernel code,
> > - * instrumentable code, or any code which may trigger an exception.
> > - *
> > - * This is intended to match the logic in irqentry_enter(), handling the kernel
> > - * mode transitions only.
> > - */
> > -static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)
> > -{
> > - regs->exit_rcu = false;
> > -
> > - if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
> > - lockdep_hardirqs_off(CALLER_ADDR0);
> > - ct_irq_enter();
> > - trace_hardirqs_off_finish();
> > -
> > - regs->exit_rcu = true;
> > - return;
> > - }
> > -
> > - lockdep_hardirqs_off(CALLER_ADDR0);
> > - rcu_irq_enter_check_tick();
> > - trace_hardirqs_off_finish();
> > -}
> > -
> > -static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
> > -{
> > - __enter_from_kernel_mode(regs);
> > - mte_check_tfsr_entry();
> > - mte_disable_tco_entry(current);
> > -}
> > -
> > -/*
> > - * Handle IRQ/context state management when exiting to kernel mode.
> > - * After this function returns it is not safe to call regular kernel code,
> > - * instrumentable code, or any code which may trigger an exception.
> > - *
> > - * This is intended to match the logic in irqentry_exit(), handling the kernel
> > - * mode transitions only, and with preemption handled elsewhere.
> > - */
> > -static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)
> > -{
> > - lockdep_assert_irqs_disabled();
> > -
> > - if (interrupts_enabled(regs)) {
> > - if (regs->exit_rcu) {
> > - trace_hardirqs_on_prepare();
> > - lockdep_hardirqs_on_prepare();
> > - ct_irq_exit();
> > - lockdep_hardirqs_on(CALLER_ADDR0);
> > - return;
> > - }
> > -
> > - trace_hardirqs_on();
> > - } else {
> > - if (regs->exit_rcu)
> > - ct_irq_exit();
> > - }
> > -}
> > -
> > -static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
> > -{
> > - mte_check_tfsr_exit();
> > - __exit_to_kernel_mode(regs);
> > -}
> > -
> > -/*
> > - * Handle IRQ/context state management when entering from user mode.
> > - * Before this function is called it is not safe to call regular kernel code,
> > - * instrumentable code, or any code which may trigger an exception.
> > - */
> > -static __always_inline void __enter_from_user_mode(void)
> > +static __always_inline void exit_to_user_mode_wrapper(struct pt_regs *regs)
> > {
> > - lockdep_hardirqs_off(CALLER_ADDR0);
> > - CT_WARN_ON(ct_state() != CONTEXT_USER);
> > - user_exit_irqoff();
> > - trace_hardirqs_off_finish();
> > - mte_disable_tco_entry(current);
> > -}
> > -
> > -static __always_inline void enter_from_user_mode(struct pt_regs *regs)
> > -{
> > - __enter_from_user_mode();
> > -}
> > -
> > -/*
> > - * Handle IRQ/context state management when exiting to user mode.
> > - * After this function returns it is not safe to call regular kernel code,
> > - * instrumentable code, or any code which may trigger an exception.
> > - */
> > -static __always_inline void __exit_to_user_mode(void)
> > -{
> > - trace_hardirqs_on_prepare();
> > - lockdep_hardirqs_on_prepare();
> > - user_enter_irqoff();
> > - lockdep_hardirqs_on(CALLER_ADDR0);
> > -}
> > -
> > -static void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags)
> > -{
> > - do {
> > - local_irq_enable();
> > -
> > - if (thread_flags & _TIF_NEED_RESCHED)
> > - schedule();
> > -
> > - if (thread_flags & _TIF_UPROBE)
> > - uprobe_notify_resume(regs);
> > -
> > - if (thread_flags & _TIF_MTE_ASYNC_FAULT) {
> > - clear_thread_flag(TIF_MTE_ASYNC_FAULT);
> > - send_sig_fault(SIGSEGV, SEGV_MTEAERR,
> > - (void __user *)NULL, current);
> > - }
> > -
> > - if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
> > - do_signal(regs);
> > -
> > - if (thread_flags & _TIF_NOTIFY_RESUME)
> > - resume_user_mode_work(regs);
> > -
> > - if (thread_flags & _TIF_FOREIGN_FPSTATE)
> > - fpsimd_restore_current_state();
> > -
> > - local_irq_disable();
> > - thread_flags = read_thread_flags();
> > - } while (thread_flags & _TIF_WORK_MASK);
> > -}
> > -
> > -static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)
> > -{
> > - unsigned long flags;
> > -
> > local_irq_disable();
> > -
> > - flags = read_thread_flags();
> > - if (unlikely(flags & _TIF_WORK_MASK))
> > - do_notify_resume(regs, flags);
> > -
> > - local_daif_mask();
> > -
> > - lockdep_sys_exit();
> > -}
> > -
> > -static __always_inline void exit_to_user_mode(struct pt_regs *regs)
> > -{
> > - exit_to_user_mode_prepare(regs);
> > - mte_check_tfsr_exit();
> > - __exit_to_user_mode();
> > + irqentry_exit_to_user_mode(regs);
> > }
> >
> > asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
> > {
> > - exit_to_user_mode(regs);
> > -}
> > -
> > -/*
> > - * Handle IRQ/context state management when entering an NMI from user/kernel
> > - * mode. Before this function is called it is not safe to call regular kernel
> > - * code, instrumentable code, or any code which may trigger an exception.
> > - */
> > -static void noinstr arm64_enter_nmi(struct pt_regs *regs)
> > -{
> > - regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
> > -
> > - __nmi_enter();
> > - lockdep_hardirqs_off(CALLER_ADDR0);
> > - lockdep_hardirq_enter();
> > - ct_nmi_enter();
> > -
> > - trace_hardirqs_off_finish();
> > - ftrace_nmi_enter();
> > -}
> > -
> > -/*
> > - * Handle IRQ/context state management when exiting an NMI from user/kernel
> > - * mode. After this function returns it is not safe to call regular kernel
> > - * code, instrumentable code, or any code which may trigger an exception.
> > - */
> > -static void noinstr arm64_exit_nmi(struct pt_regs *regs)
> > -{
> > - bool restore = regs->lockdep_hardirqs;
> > -
> > - ftrace_nmi_exit();
> > - if (restore) {
> > - trace_hardirqs_on_prepare();
> > - lockdep_hardirqs_on_prepare();
> > - }
> > -
> > - ct_nmi_exit();
> > - lockdep_hardirq_exit();
> > - if (restore)
> > - lockdep_hardirqs_on(CALLER_ADDR0);
> > - __nmi_exit();
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > /*
> > @@ -259,48 +74,6 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
> > lockdep_hardirqs_on(CALLER_ADDR0);
> > }
> >
> > -#ifdef CONFIG_PREEMPT_DYNAMIC
> > -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
> > -#define need_irq_preemption() \
> > - (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
> > -#else
> > -#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION))
> > -#endif
> > -
> > -static void __sched arm64_preempt_schedule_irq(void)
> > -{
> > - if (!need_irq_preemption())
> > - return;
> > -
> > - /*
> > - * Note: thread_info::preempt_count includes both thread_info::count
> > - * and thread_info::need_resched, and is not equivalent to
> > - * preempt_count().
> > - */
> > - if (READ_ONCE(current_thread_info()->preempt_count) != 0)
> > - return;
> > -
> > - /*
> > - * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
> > - * priority masking is used the GIC irqchip driver will clear DAIF.IF
> > - * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
> > - * DAIF we must have handled an NMI, so skip preemption.
> > - */
> > - if (system_uses_irq_prio_masking() && read_sysreg(daif))
> > - return;
> > -
> > - /*
> > - * Preempting a task from an IRQ means we leave copies of PSTATE
> > - * on the stack. cpufeature's enable calls may modify PSTATE, but
> > - * resuming one of these preempted tasks would undo those changes.
> > - *
> > - * Only allow a task to be preempted once cpufeatures have been
> > - * enabled.
> > - */
> > - if (system_capabilities_finalized())
> > - preempt_schedule_irq();
> > -}
> > -
> > static void do_interrupt_handler(struct pt_regs *regs,
> > void (*handler)(struct pt_regs *))
> > {
> > @@ -320,7 +93,7 @@ extern void (*handle_arch_fiq)(struct pt_regs *);
> > static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
> > unsigned long esr)
> > {
> > - arm64_enter_nmi(regs);
> > + irqentry_nmi_enter(regs);
> >
> > console_verbose();
> >
> > @@ -426,41 +199,43 @@ UNHANDLED(el1t, 64, error)
> > static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
> > {
> > unsigned long far = read_sysreg(far_el1);
> > + irqentry_state_t state = irqentry_enter(regs);
> >
> > - enter_from_kernel_mode(regs);
> > local_daif_inherit(regs);
> > do_mem_abort(far, esr, regs);
> > local_daif_mask();
> > - exit_to_kernel_mode(regs);
> > + irqentry_exit(regs, state);
> > }
> >
> > static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
> > {
> > unsigned long far = read_sysreg(far_el1);
> > + irqentry_state_t state = irqentry_enter(regs);
> >
> > - enter_from_kernel_mode(regs);
> > local_daif_inherit(regs);
> > do_sp_pc_abort(far, esr, regs);
> > local_daif_mask();
> > - exit_to_kernel_mode(regs);
> > + irqentry_exit(regs, state);
> > }
> >
> > static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_kernel_mode(regs);
> > + irqentry_state_t state = irqentry_enter(regs);
> > +
> > local_daif_inherit(regs);
> > do_el1_undef(regs, esr);
> > local_daif_mask();
> > - exit_to_kernel_mode(regs);
> > + irqentry_exit(regs, state);
> > }
> >
> > static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_kernel_mode(regs);
> > + irqentry_state_t state = irqentry_enter(regs);
> > +
> > local_daif_inherit(regs);
> > do_el1_bti(regs, esr);
> > local_daif_mask();
> > - exit_to_kernel_mode(regs);
> > + irqentry_exit(regs, state);
> > }
> >
> > static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
> > @@ -475,11 +250,12 @@ static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
> >
> > static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_kernel_mode(regs);
> > + irqentry_state_t state = irqentry_enter(regs);
> > +
> > local_daif_inherit(regs);
> > do_el1_fpac(regs, esr);
> > local_daif_mask();
> > - exit_to_kernel_mode(regs);
> > + irqentry_exit(regs, state);
> > }
> >
> > asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
> > @@ -522,23 +298,22 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
> > static __always_inline void __el1_pnmi(struct pt_regs *regs,
> > void (*handler)(struct pt_regs *))
> > {
> > - arm64_enter_nmi(regs);
> > + irqentry_state_t state = irqentry_nmi_enter(regs);
> > +
> > do_interrupt_handler(regs, handler);
> > - arm64_exit_nmi(regs);
> > + irqentry_nmi_exit(regs, state);
> > }
> >
> > static __always_inline void __el1_irq(struct pt_regs *regs,
> > void (*handler)(struct pt_regs *))
> > {
> > - enter_from_kernel_mode(regs);
> > + irqentry_state_t state = irqentry_enter(regs);
> >
> > irq_enter_rcu();
> > do_interrupt_handler(regs, handler);
> > irq_exit_rcu();
> >
> > - arm64_preempt_schedule_irq();
> > -
> > - exit_to_kernel_mode(regs);
> > + irqentry_exit(regs, state);
> > }
> > static void noinstr el1_interrupt(struct pt_regs *regs,
> > void (*handler)(struct pt_regs *))
> > @@ -564,21 +339,22 @@ asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
> > asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
> > {
> > unsigned long esr = read_sysreg(esr_el1);
> > + irqentry_state_t state;
> >
> > local_daif_restore(DAIF_ERRCTX);
> > - arm64_enter_nmi(regs);
> > + state = irqentry_nmi_enter(regs);
> > do_serror(regs, esr);
> > - arm64_exit_nmi(regs);
> > + irqentry_nmi_exit(regs, state);
> > }
> >
> > static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
> > {
> > unsigned long far = read_sysreg(far_el1);
> >
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_mem_abort(far, esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
> > @@ -593,50 +369,50 @@ static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
> > if (!is_ttbr0_addr(far))
> > arm64_apply_bp_hardening();
> >
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_mem_abort(far, esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_fpsimd_acc(esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_sve_acc(esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_sme_acc(esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_fpsimd_exc(esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_el0_sys(esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
> > @@ -646,50 +422,50 @@ static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
> > if (!is_ttbr0_addr(instruction_pointer(regs)))
> > arm64_apply_bp_hardening();
> >
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_sp_pc_abort(far, esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_sp_pc_abort(regs->sp, esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_el0_undef(regs, esr);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_bti(struct pt_regs *regs)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_el0_bti(regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_el0_mops(regs, esr);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > bad_el0_sync(regs, 0, esr);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
> > @@ -697,28 +473,28 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
> > /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
> > unsigned long far = read_sysreg(far_el1);
> >
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > do_debug_exception(far, esr, regs);
> > local_daif_restore(DAIF_PROCCTX);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_svc(struct pt_regs *regs)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > cortex_a76_erratum_1463225_svc_handler();
> > fp_user_discard();
> > local_daif_restore(DAIF_PROCCTX);
> > do_el0_svc(regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_el0_fpac(regs, esr);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
> > @@ -783,7 +559,7 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
> > static void noinstr el0_interrupt(struct pt_regs *regs,
> > void (*handler)(struct pt_regs *))
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> >
> > write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
> >
> > @@ -794,7 +570,7 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
> > do_interrupt_handler(regs, handler);
> > irq_exit_rcu();
> >
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
> > @@ -820,14 +596,15 @@ asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
> > static void noinstr __el0_error_handler_common(struct pt_regs *regs)
> > {
> > unsigned long esr = read_sysreg(esr_el1);
> > + irqentry_state_t state_nmi;
> >
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_ERRCTX);
> > - arm64_enter_nmi(regs);
> > + state_nmi = irqentry_nmi_enter(regs);
> > do_serror(regs, esr);
> > - arm64_exit_nmi(regs);
> > + irqentry_nmi_exit(regs, state_nmi);
> > local_daif_restore(DAIF_PROCCTX);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
> > @@ -838,19 +615,19 @@ asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
> > #ifdef CONFIG_COMPAT
> > static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > local_daif_restore(DAIF_PROCCTX);
> > do_el0_cp15(esr, regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > static void noinstr el0_svc_compat(struct pt_regs *regs)
> > {
> > - enter_from_user_mode(regs);
> > + irqentry_enter_from_user_mode(regs);
> > cortex_a76_erratum_1463225_svc_handler();
> > local_daif_restore(DAIF_PROCCTX);
> > do_el0_svc_compat(regs);
> > - exit_to_user_mode(regs);
> > + exit_to_user_mode_wrapper(regs);
> > }
> >
> > asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
> > @@ -924,7 +701,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
> > unsigned long esr = read_sysreg(esr_el1);
> > unsigned long far = read_sysreg(far_el1);
> >
> > - arm64_enter_nmi(regs);
> > + irqentry_nmi_enter(regs);
> > panic_bad_stack(regs, esr, far);
> > }
> > #endif /* CONFIG_VMAP_STACK */
> > @@ -933,6 +710,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
> > asmlinkage noinstr unsigned long
> > __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
> > {
> > + irqentry_state_t state;
> > unsigned long ret;
> >
> > /*
> > @@ -957,9 +735,9 @@ __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
> > else if (cpu_has_pan())
> > set_pstate_pan(0);
> >
> > - arm64_enter_nmi(regs);
> > + state = irqentry_nmi_enter(regs);
> > ret = do_sdei_event(regs, arg);
> > - arm64_exit_nmi(regs);
> > + irqentry_nmi_exit(regs, state);
> >
> > return ret;
> > }
> > diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
> > index 60fd85d5119d..9cc0f450fa76 100644
> > --- a/arch/arm64/kernel/ptrace.c
> > +++ b/arch/arm64/kernel/ptrace.c
> > @@ -41,9 +41,6 @@
> > #include <asm/traps.h>
> > #include <asm/system_misc.h>
> >
> > -#define CREATE_TRACE_POINTS
> > -#include <trace/events/syscalls.h>
> > -
> > struct pt_regs_offset {
> > const char *name;
> > int offset;
> > @@ -2179,104 +2176,6 @@ long arch_ptrace(struct task_struct *child, long request,
> > return ptrace_request(child, request, addr, data);
> > }
> >
> > -enum ptrace_syscall_dir {
> > - PTRACE_SYSCALL_ENTER = 0,
> > - PTRACE_SYSCALL_EXIT,
> > -};
> > -
> > -static void report_syscall_enter(struct pt_regs *regs)
> > -{
> > - int regno;
> > - unsigned long saved_reg;
> > -
> > - /*
> > - * We have some ABI weirdness here in the way that we handle syscall
> > - * exit stops because we indicate whether or not the stop has been
> > - * signalled from syscall entry or syscall exit by clobbering a general
> > - * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
> > - * and restoring its old value after the stop. This means that:
> > - *
> > - * - Any writes by the tracer to this register during the stop are
> > - * ignored/discarded.
> > - *
> > - * - The actual value of the register is not available during the stop,
> > - * so the tracer cannot save it and restore it later.
> > - *
> > - * - Syscall stops behave differently to seccomp and pseudo-step traps
> > - * (the latter do not nobble any registers).
> > - */
> > - regno = (is_compat_task() ? 12 : 7);
> > - saved_reg = regs->regs[regno];
> > - regs->regs[regno] = PTRACE_SYSCALL_ENTER;
> > -
> > - if (ptrace_report_syscall_entry(regs))
> > - forget_syscall(regs);
> > - regs->regs[regno] = saved_reg;
> > -}
> > -
> > -static void report_syscall_exit(struct pt_regs *regs)
> > -{
> > - int regno;
> > - unsigned long saved_reg;
> > -
> > - /* See comment for report_syscall_enter() */
> > - regno = (is_compat_task() ? 12 : 7);
> > - saved_reg = regs->regs[regno];
> > - regs->regs[regno] = PTRACE_SYSCALL_EXIT;
> > -
> > - if (!test_thread_flag(TIF_SINGLESTEP)) {
> > - ptrace_report_syscall_exit(regs, 0);
> > - regs->regs[regno] = saved_reg;
> > - } else {
> > - regs->regs[regno] = saved_reg;
> > -
> > - /*
> > - * Signal a pseudo-step exception since we are stepping but
> > - * tracer modifications to the registers may have rewound the
> > - * state machine.
> > - */
> > - ptrace_report_syscall_exit(regs, 1);
> > - }
> > -}
> > -
> > -int syscall_trace_enter(struct pt_regs *regs)
> > -{
> > - unsigned long flags = read_thread_flags();
> > -
> > - if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
> > - report_syscall_enter(regs);
> > - if (flags & _TIF_SYSCALL_EMU)
> > - return NO_SYSCALL;
> > - }
> > -
> > - /* Do the secure computing after ptrace; failures should be fast. */
> > - if (secure_computing() == -1)
> > - return NO_SYSCALL;
> > -
> > - if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
> > - trace_sys_enter(regs, regs->syscallno);
> > -
> > - audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
> > - regs->regs[2], regs->regs[3]);
> > -
> > - return regs->syscallno;
> > -}
> > -
> > -void syscall_trace_exit(struct pt_regs *regs)
> > -{
> > - unsigned long flags = read_thread_flags();
> > -
> > - audit_syscall_exit(regs);
> > -
> > - if (flags & _TIF_SYSCALL_TRACEPOINT)
> > - trace_sys_exit(regs, syscall_get_return_value(current, regs));
> > -
> > - if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP))
> > - report_syscall_exit(regs);
> > -
> > - rseq_syscall(regs);
> > -}
> > -
> > /*
> > * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
> > * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
> > diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
> > index 4a77f4976e11..2982f6db6d96 100644
> > --- a/arch/arm64/kernel/signal.c
> > +++ b/arch/arm64/kernel/signal.c
> > @@ -19,6 +19,7 @@
> > #include <linux/ratelimit.h>
> > #include <linux/rseq.h>
> > #include <linux/syscalls.h>
> > +#include <linux/entry-common.h>
> >
> > #include <asm/daifflags.h>
> > #include <asm/debug-monitors.h>
> > @@ -1266,7 +1267,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
> > * the kernel can handle, and then we build all the user-level signal handling
> > * stack-frames in one go after that.
> > */
> > -void do_signal(struct pt_regs *regs)
> > +void arch_do_signal_or_restart(struct pt_regs *regs)
> > {
> > unsigned long continue_addr = 0, restart_addr = 0;
> > int retval = 0;
> > diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c
> > index ad198262b981..160ac9d15c27 100644
> > --- a/arch/arm64/kernel/syscall.c
> > +++ b/arch/arm64/kernel/syscall.c
> > @@ -7,6 +7,7 @@
> > #include <linux/ptrace.h>
> > #include <linux/randomize_kstack.h>
> > #include <linux/syscalls.h>
> > +#include <linux/entry-common.h>
> >
> > #include <asm/debug-monitors.h>
> > #include <asm/exception.h>
> > @@ -66,14 +67,15 @@ static void invoke_syscall(struct pt_regs *regs, unsigned int scno,
> > choose_random_kstack_offset(get_random_u16() & 0x1FF);
> > }
> >
> > -static inline bool has_syscall_work(unsigned long flags)
> > +static inline bool has_syscall_work(unsigned long work)
> > {
> > - return unlikely(flags & _TIF_SYSCALL_WORK);
> > + return unlikely(work & SYSCALL_WORK_ENTER);
> > }
> >
> > static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> > const syscall_fn_t syscall_table[])
> > {
> > + unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
> > unsigned long flags = read_thread_flags();
> >
> > regs->orig_x0 = regs->regs[0];
> > @@ -107,7 +109,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> > return;
> > }
> >
> > - if (has_syscall_work(flags)) {
> > + if (has_syscall_work(work)) {
> > /*
> > * The de-facto standard way to skip a system call using ptrace
> > * is to set the system call to -1 (NO_SYSCALL) and set x0 to a
> > @@ -125,7 +127,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> > */
> > if (scno == NO_SYSCALL)
> > syscall_set_return_value(current, regs, -ENOSYS, 0);
> > - scno = syscall_trace_enter(regs);
> > + scno = syscall_trace_enter(regs, regs->syscallno, work);
> > if (scno == NO_SYSCALL)
> > goto trace_exit;
> > }
> > @@ -137,14 +139,14 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> > * check again. However, if we were tracing entry, then we always trace
> > * exit regardless, as the old entry assembly did.
> > */
> > - if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
> > - flags = read_thread_flags();
> > - if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
> > + if (!has_syscall_work(work) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
> > + work = READ_ONCE(current_thread_info()->syscall_work);
> > + if (!has_syscall_work(work) && !report_single_step(work))
> > return;
> > }
> >
> > trace_exit:
> > - syscall_trace_exit(regs);
> > + syscall_exit_work(regs, work);
> > }
> >
> > void do_el0_svc(struct pt_regs *regs)
On 29/06/2024 10:56, Jinjie Ruan wrote:
> Currently, x86, Riscv, Loongarch use the generic entry. Convert arm64
> to use the generic entry infrastructure from kernel/entry/*. The generic
> entry makes maintainers' work easier and codes more elegant, which also
> removed duplicate 150+ LOC. The changes are below:
>
> - Remove TIF_SYSCALL_* flag, _TIF_WORK_MASK, _TIF_SYSCALL_WORK
>
> - Remove syscall_trace_enter/exit() and use generic one.
>
> - Remove *enter_from/exit_to_kernel_mode(), and wrap with generic
> irqentry_enter/exit().
>
> - Remove *enter_from/exit_to_user_mode(), and wrap with generic
> irqentry_enter_from/exit_to_user_mode().
>
> - Remove arm64_enter/exit_nmi() and use generic irqentry_nmi_enter/exit().
>
> - Remove PREEMPT_DYNAMIC code, as generic entry will do it ok by
> implementing arch_irqentry_exit_need_resched().
This is looking good to me overall, moving to using the generic helpers
is a clear improvement. I've tried my best to check that the new
implementation is functionally equivalent to the old. Ignoring additions
in the generic code (such as additional instrumentation_{begin,end}()
pairs or assertions), this seems to be the case, with one exception. The
MTE operations are done in a slightly different order on entry:
* mte_disable_tco_entry() was called *after* the generic lockdep/CT
functions in __enter_from_user_mode(). It is now called before those
generic operations - arch_enter_from_user_mode() called at the beginning
of enter_from_user_mode().
* Similarly mte_disable_tco_entry() and mte_check_tfsr_entry() was
called after the generic operations in enter_from_kernel_mode(), and
they are now called after - arch_enter_from_kernel_mode() called at the
beginning of irqentry_enter().
I am not under the impression that these ordering changes are
problematic, but I may be missing something.
> [...]
>
> -/*
> - * Handle IRQ/context state management when entering from kernel mode.
> - * Before this function is called it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - *
> - * This is intended to match the logic in irqentry_enter(), handling the kernel
> - * mode transitions only.
> - */
> -static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)
> -{
> - regs->exit_rcu = false;
exit_rcu in struct pt_regs is unused now that these functions are gone
so it can be removed.
> [...]
>
> @@ -259,48 +74,6 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
arm64_{enter,exit}_el1_dbg() have apparently no generic counterparts we
can replace them with, but maybe we could align them with the generic
functions some more? Specifically, I'm thinking about making them
return/take an irqentry_state_t just like irqentry_nmi_{enter,exit}().
This way we can get rid of struct pt_regs::lockdep_hardirqs, which is
now only used by those functions.
> lockdep_hardirqs_on(CALLER_ADDR0);
> }
>
> -#ifdef CONFIG_PREEMPT_DYNAMIC
> -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
The key is declared in asm/preempt.h, that declaration should also be
removed.
Kevin
> [...]
On 2024/8/20 19:43, Kevin Brodsky wrote:
> On 29/06/2024 10:56, Jinjie Ruan wrote:
>> Currently, x86, Riscv, Loongarch use the generic entry. Convert arm64
>> to use the generic entry infrastructure from kernel/entry/*. The generic
>> entry makes maintainers' work easier and codes more elegant, which also
>> removed duplicate 150+ LOC. The changes are below:
>>
>> - Remove TIF_SYSCALL_* flag, _TIF_WORK_MASK, _TIF_SYSCALL_WORK
>>
>> - Remove syscall_trace_enter/exit() and use generic one.
>>
>> - Remove *enter_from/exit_to_kernel_mode(), and wrap with generic
>> irqentry_enter/exit().
>>
>> - Remove *enter_from/exit_to_user_mode(), and wrap with generic
>> irqentry_enter_from/exit_to_user_mode().
>>
>> - Remove arm64_enter/exit_nmi() and use generic irqentry_nmi_enter/exit().
>>
>> - Remove PREEMPT_DYNAMIC code, as generic entry will do it ok by
>> implementing arch_irqentry_exit_need_resched().
>
> This is looking good to me overall, moving to using the generic helpers
> is a clear improvement. I've tried my best to check that the new
Thank you very much for the review, It does make the code more clear.
> implementation is functionally equivalent to the old. Ignoring additions
> in the generic code (such as additional instrumentation_{begin,end}()
> pairs or assertions), this seems to be the case, with one exception. The
> MTE operations are done in a slightly different order on entry:
>
> * mte_disable_tco_entry() was called *after* the generic lockdep/CT
> functions in __enter_from_user_mode(). It is now called before those
> generic operations - arch_enter_from_user_mode() called at the beginning
> of enter_from_user_mode().
The most special for arm64 are these MTE operations, the comment for
__enter_from_kernel_mode() and __enter_from_user_mode() said:
" Handle IRQ/context state management when entering from user/kernel
mode. Before this function is called it is not safe to call regular
kernel code "
I guess it is the reason why mte_disable_tco_entry() was called *after*
the generic lockdep/CT functions, but the first version of commit
38ddf7dafaeaf ("arm64: mte: avoid clearing PSTATE.TCO on entry unless
necessary") call it in arch/arm64/kernel/entry.S, so I think the order
is not so important.
>
> * Similarly mte_disable_tco_entry() and mte_check_tfsr_entry() was
> called after the generic operations in enter_from_kernel_mode(), and
> they are now called after - arch_enter_from_kernel_mode() called at the
> beginning of irqentry_enter().
This can be adjusted to be consistent.
>
> I am not under the impression that these ordering changes are
> problematic, but I may be missing something.>
>> [...]
>>
>> -/*
>> - * Handle IRQ/context state management when entering from kernel mode.
>> - * Before this function is called it is not safe to call regular kernel code,
>> - * instrumentable code, or any code which may trigger an exception.
>> - *
>> - * This is intended to match the logic in irqentry_enter(), handling the kernel
>> - * mode transitions only.
>> - */
>> -static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)
>> -{
>> - regs->exit_rcu = false;
>
> exit_rcu in struct pt_regs is unused now that these functions are gone
> so it can be removed.
It is also consistent with ARM64.
>
>> [...]
>>
>> @@ -259,48 +74,6 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
>
> arm64_{enter,exit}_el1_dbg() have apparently no generic counterparts we
> can replace them with, but maybe we could align them with the generic
> functions some more? Specifically, I'm thinking about making them
> return/take an irqentry_state_t just like irqentry_nmi_{enter,exit}().
> This way we can get rid of struct pt_regs::lockdep_hardirqs, which is
> now only used by those functions.
>
>> lockdep_hardirqs_on(CALLER_ADDR0);
>> }
>>
>> -#ifdef CONFIG_PREEMPT_DYNAMIC
>> -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
>
> The key is declared in asm/preempt.h, that declaration should also be
> removed.
Yes, you are right! commit 1b2d3451ee50 ("arm64: Support
PREEMPT_DYNAMIC") mentioned it:
"Since arm64 does not yet use the generic entry code, we must define
our own `sk_dynamic_irqentry_exit_cond_resched`, which will be
enabled/disabled by the common code in kernel/sched/core.c. All other
preemption functions and associated static keys are defined there."
So if arm64 switched to generic entry, the code can be removed.
>
> Kevin
>
>> [...]
>
>
On 22/08/2024 14:30, Jinjie Ruan wrote:
> On 2024/8/20 19:43, Kevin Brodsky wrote:
>> On 29/06/2024 10:56, Jinjie Ruan wrote:
>>> Currently, x86, Riscv, Loongarch use the generic entry. Convert arm64
>>> to use the generic entry infrastructure from kernel/entry/*. The generic
>>> entry makes maintainers' work easier and codes more elegant, which also
>>> removed duplicate 150+ LOC. The changes are below:
>>>
>>> - Remove TIF_SYSCALL_* flag, _TIF_WORK_MASK, _TIF_SYSCALL_WORK
>>>
>>> - Remove syscall_trace_enter/exit() and use generic one.
>>>
>>> - Remove *enter_from/exit_to_kernel_mode(), and wrap with generic
>>> irqentry_enter/exit().
>>>
>>> - Remove *enter_from/exit_to_user_mode(), and wrap with generic
>>> irqentry_enter_from/exit_to_user_mode().
>>>
>>> - Remove arm64_enter/exit_nmi() and use generic irqentry_nmi_enter/exit().
>>>
>>> - Remove PREEMPT_DYNAMIC code, as generic entry will do it ok by
>>> implementing arch_irqentry_exit_need_resched().
>> This is looking good to me overall, moving to using the generic helpers
>> is a clear improvement. I've tried my best to check that the new
> Thank you very much for the review, It does make the code more clear.
>
>> implementation is functionally equivalent to the old. Ignoring additions
>> in the generic code (such as additional instrumentation_{begin,end}()
>> pairs or assertions), this seems to be the case, with one exception. The
>> MTE operations are done in a slightly different order on entry:
>>
>> * mte_disable_tco_entry() was called *after* the generic lockdep/CT
>> functions in __enter_from_user_mode(). It is now called before those
>> generic operations - arch_enter_from_user_mode() called at the beginning
>> of enter_from_user_mode().
> The most special for arm64 are these MTE operations, the comment for
> __enter_from_kernel_mode() and __enter_from_user_mode() said:
>
> " Handle IRQ/context state management when entering from user/kernel
> mode. Before this function is called it is not safe to call regular
> kernel code "
>
> I guess it is the reason why mte_disable_tco_entry() was called *after*
> the generic lockdep/CT functions, but the first version of commit
> 38ddf7dafaeaf ("arm64: mte: avoid clearing PSTATE.TCO on entry unless
> necessary") call it in arch/arm64/kernel/entry.S, so I think the order
> is not so important.
Good point, before that commit TCO was disabled before any generic
exception code was run so changing the ordering of
mte_disable_tco_entry() is unlikely to be a problem (the current
implementation doesn't do anything complex).
>
>> * Similarly mte_disable_tco_entry() and mte_check_tfsr_entry() was
>> called after the generic operations in enter_from_kernel_mode(), and
>> they are now called after - arch_enter_from_kernel_mode() called at the
>> beginning of irqentry_enter().
> This can be adjusted to be consistent.
I wonder if moving mte_check_tfsr_entry() is as harmless as
mte_disable_tco_entry(), because the former can cause a stack dump. Is
that safe to do before the lockdep/RCU handling? I would guess so but
would be good for someone to confirm.
Kevin
Gentle ping.
On 2024/6/29 16:56, Jinjie Ruan wrote:
> Currently, x86, Riscv, Loongarch use the generic entry. Convert arm64
> to use the generic entry infrastructure from kernel/entry/*. The generic
> entry makes maintainers' work easier and codes more elegant, which also
> removed duplicate 150+ LOC. The changes are below:
>
> - Remove TIF_SYSCALL_* flag, _TIF_WORK_MASK, _TIF_SYSCALL_WORK
>
> - Remove syscall_trace_enter/exit() and use generic one.
>
> - Remove *enter_from/exit_to_kernel_mode(), and wrap with generic
> irqentry_enter/exit().
>
> - Remove *enter_from/exit_to_user_mode(), and wrap with generic
> irqentry_enter_from/exit_to_user_mode().
>
> - Remove arm64_enter/exit_nmi() and use generic irqentry_nmi_enter/exit().
>
> - Remove PREEMPT_DYNAMIC code, as generic entry will do it ok by
> implementing arch_irqentry_exit_need_resched().
>
> Tested ok with following test cases on Qemu cortex-a53 and HiSilicon
> Kunpeng-920:
>
> - Run `perf top` command
>
> - Switch between different `dynamic preempt` mode
>
> - Use `pseudo nmi`
>
> - stress-ng CPU stress test.
>
> - MTE test case in Documentation/arch/arm64/memory-tagging-extension.rst
> and all test cases in tools/testing/selftests/arm64/mte/* (Only Qemu).
>
> Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com>
> Tested-by: Jinjie Ruan <ruanjinjie@huawei.com>
> Tested-by: Kees Cook <kees@kernel.org>
> ---
> v3:
> - Test the MTE test cases.
> - Handle forget_syscall() in arch_post_report_syscall_entry()
> - Rebased on the previous patch, so move the arch funcs to entry-common.h.
> - Change __always_inline() to inline for arch_enter_from_kernel_mode() and
> arch_exit_to_kernel_mode_prepare(), add inline for the other arch funcs.
> - Remove unused signal.h for entry-common.h.
>
> v2:
> - Add Tested-by.
> - Rebased on the refactored report_syscall() code.
> - Add comment for arch_prepare_report_syscall_exit().
> - Update the commit message.
> - Adjust entry-common.h header file inclusion to alphabetical order.
> ---
> arch/arm64/Kconfig | 1 +
> arch/arm64/include/asm/entry-common.h | 172 ++++++++++++
> arch/arm64/include/asm/syscall.h | 6 +-
> arch/arm64/include/asm/thread_info.h | 23 +-
> arch/arm64/kernel/entry-common.c | 368 +++++---------------------
> arch/arm64/kernel/ptrace.c | 101 -------
> arch/arm64/kernel/signal.c | 3 +-
> arch/arm64/kernel/syscall.c | 18 +-
> 8 files changed, 264 insertions(+), 428 deletions(-)
> create mode 100644 arch/arm64/include/asm/entry-common.h
>
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 5d91259ee7b5..e6ccc5ea06fe 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -138,6 +138,7 @@ config ARM64
> select GENERIC_CPU_DEVICES
> select GENERIC_CPU_VULNERABILITIES
> select GENERIC_EARLY_IOREMAP
> + select GENERIC_ENTRY
> select GENERIC_IDLE_POLL_SETUP
> select GENERIC_IOREMAP
> select GENERIC_IRQ_IPI
> diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h
> new file mode 100644
> index 000000000000..a14d62b2eb69
> --- /dev/null
> +++ b/arch/arm64/include/asm/entry-common.h
> @@ -0,0 +1,172 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +
> +#ifndef _ASM_ARM64_ENTRY_COMMON_H
> +#define _ASM_ARM64_ENTRY_COMMON_H
> +
> +#include <linux/thread_info.h>
> +
> +#include <asm/daifflags.h>
> +#include <asm/fpsimd.h>
> +#include <asm/mte.h>
> +#include <asm/stacktrace.h>
> +
> +enum ptrace_syscall_dir {
> + PTRACE_SYSCALL_ENTER = 0,
> + PTRACE_SYSCALL_EXIT,
> +};
> +
> +#define ARCH_EXIT_TO_USER_MODE_WORK (_TIF_MTE_ASYNC_FAULT | _TIF_FOREIGN_FPSTATE)
> +
> +static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs)
> +{
> + mte_disable_tco_entry(current);
> +}
> +
> +#define arch_enter_from_user_mode arch_enter_from_user_mode
> +
> +static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs,
> + unsigned long ti_work)
> +{
> + if (ti_work & _TIF_MTE_ASYNC_FAULT) {
> + clear_thread_flag(TIF_MTE_ASYNC_FAULT);
> + send_sig_fault(SIGSEGV, SEGV_MTEAERR, (void __user *)NULL, current);
> + }
> +
> + if (ti_work & _TIF_FOREIGN_FPSTATE)
> + fpsimd_restore_current_state();
> +}
> +
> +#define arch_exit_to_user_mode_work arch_exit_to_user_mode_work
> +
> +static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
> + unsigned long ti_work)
> +{
> + local_daif_mask();
> + mte_check_tfsr_exit();
> +}
> +
> +#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
> +
> +static inline void arch_enter_from_kernel_mode(struct pt_regs *regs)
> +{
> + mte_check_tfsr_entry();
> + mte_disable_tco_entry(current);
> +}
> +
> +#define arch_enter_from_kernel_mode arch_enter_from_kernel_mode
> +
> +static inline void arch_exit_to_kernel_mode_prepare(struct pt_regs *regs)
> +{
> + mte_check_tfsr_exit();
> +}
> +
> +#define arch_exit_to_kernel_mode_prepare arch_exit_to_kernel_mode_prepare
> +
> +static inline bool arch_irqentry_exit_need_resched(void)
> +{
> + /*
> + * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
> + * priority masking is used the GIC irqchip driver will clear DAIF.IF
> + * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
> + * DAIF we must have handled an NMI, so skip preemption.
> + */
> + if (system_uses_irq_prio_masking() && read_sysreg(daif))
> + return false;
> +
> + /*
> + * Preempting a task from an IRQ means we leave copies of PSTATE
> + * on the stack. cpufeature's enable calls may modify PSTATE, but
> + * resuming one of these preempted tasks would undo those changes.
> + *
> + * Only allow a task to be preempted once cpufeatures have been
> + * enabled.
> + */
> + if (!system_capabilities_finalized())
> + return false;
> +
> + return true;
> +}
> +
> +#define arch_irqentry_exit_need_resched arch_irqentry_exit_need_resched
> +
> +static inline unsigned long arch_prepare_report_syscall_entry(struct pt_regs *regs)
> +{
> + unsigned long saved_reg;
> + int regno;
> +
> + /*
> + * We have some ABI weirdness here in the way that we handle syscall
> + * exit stops because we indicate whether or not the stop has been
> + * signalled from syscall entry or syscall exit by clobbering a general
> + * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
> + * and restoring its old value after the stop. This means that:
> + *
> + * - Any writes by the tracer to this register during the stop are
> + * ignored/discarded.
> + *
> + * - The actual value of the register is not available during the stop,
> + * so the tracer cannot save it and restore it later.
> + *
> + * - Syscall stops behave differently to seccomp and pseudo-step traps
> + * (the latter do not nobble any registers).
> + */
> + regno = (is_compat_task() ? 12 : 7);
> + saved_reg = regs->regs[regno];
> + regs->regs[regno] = PTRACE_SYSCALL_ENTER;
> +
> + return saved_reg;
> +}
> +
> +#define arch_prepare_report_syscall_entry arch_prepare_report_syscall_entry
> +
> +static inline void arch_post_report_syscall_entry(struct pt_regs *regs,
> + unsigned long saved_reg, long ret)
> +{
> + int regno = (is_compat_task() ? 12 : 7);
> +
> + if (ret)
> + forget_syscall(regs);
> +
> + regs->regs[regno] = saved_reg;
> +}
> +
> +#define arch_post_report_syscall_entry arch_post_report_syscall_entry
> +
> +static inline unsigned long arch_prepare_report_syscall_exit(struct pt_regs *regs,
> + unsigned long work)
> +{
> + unsigned long saved_reg;
> + int regno;
> +
> + /* See comment for arch_prepare_report_syscall_entry() */
> + regno = (is_compat_task() ? 12 : 7);
> + saved_reg = regs->regs[regno];
> + regs->regs[regno] = PTRACE_SYSCALL_EXIT;
> +
> + if (report_single_step(work)) {
> + /*
> + * Signal a pseudo-step exception since we are stepping but
> + * tracer modifications to the registers may have rewound the
> + * state machine.
> + */
> + regs->regs[regno] = saved_reg;
> + }
> +
> + return saved_reg;
> +}
> +
> +#define arch_prepare_report_syscall_exit arch_prepare_report_syscall_exit
> +
> +static inline void arch_post_report_syscall_exit(struct pt_regs *regs,
> + unsigned long saved_reg,
> + unsigned long work)
> +{
> + int regno = (is_compat_task() ? 12 : 7);
> +
> + if (!report_single_step(work))
> + regs->regs[regno] = saved_reg;
> +}
> +
> +#define arch_post_report_syscall_exit arch_post_report_syscall_exit
> +
> +#endif /* _ASM_ARM64_ENTRY_COMMON_H */
> diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h
> index ab8e14b96f68..9891b15da4c3 100644
> --- a/arch/arm64/include/asm/syscall.h
> +++ b/arch/arm64/include/asm/syscall.h
> @@ -85,7 +85,9 @@ static inline int syscall_get_arch(struct task_struct *task)
> return AUDIT_ARCH_AARCH64;
> }
>
> -int syscall_trace_enter(struct pt_regs *regs);
> -void syscall_trace_exit(struct pt_regs *regs);
> +static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
> +{
> + return false;
> +}
>
> #endif /* __ASM_SYSCALL_H */
> diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
> index e72a3bf9e563..ec5d74c53bf9 100644
> --- a/arch/arm64/include/asm/thread_info.h
> +++ b/arch/arm64/include/asm/thread_info.h
> @@ -43,6 +43,7 @@ struct thread_info {
> void *scs_sp;
> #endif
> u32 cpu;
> + unsigned long syscall_work; /* SYSCALL_WORK_ flags */
> };
>
> #define thread_saved_pc(tsk) \
> @@ -64,11 +65,6 @@ void arch_setup_new_exec(void);
> #define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */
> #define TIF_MTE_ASYNC_FAULT 5 /* MTE Asynchronous Tag Check Fault */
> #define TIF_NOTIFY_SIGNAL 6 /* signal notifications exist */
> -#define TIF_SYSCALL_TRACE 8 /* syscall trace active */
> -#define TIF_SYSCALL_AUDIT 9 /* syscall auditing */
> -#define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */
> -#define TIF_SECCOMP 11 /* syscall secure computing */
> -#define TIF_SYSCALL_EMU 12 /* syscall emulation active */
> #define TIF_MEMDIE 18 /* is terminating due to OOM killer */
> #define TIF_FREEZE 19
> #define TIF_RESTORE_SIGMASK 20
> @@ -86,27 +82,12 @@ void arch_setup_new_exec(void);
> #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
> #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
> #define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)
> -#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
> -#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
> -#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
> -#define _TIF_SECCOMP (1 << TIF_SECCOMP)
> -#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
> -#define _TIF_UPROBE (1 << TIF_UPROBE)
> -#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
> +#define _TIF_UPROBE (1 << TIF_UPROBE)
> #define _TIF_32BIT (1 << TIF_32BIT)
> #define _TIF_SVE (1 << TIF_SVE)
> #define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT)
> #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
>
> -#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
> - _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
> - _TIF_UPROBE | _TIF_MTE_ASYNC_FAULT | \
> - _TIF_NOTIFY_SIGNAL)
> -
> -#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
> - _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
> - _TIF_SYSCALL_EMU)
> -
> #ifdef CONFIG_SHADOW_CALL_STACK
> #define INIT_SCS \
> .scs_base = init_shadow_call_stack, \
> diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
> index b77a15955f28..ac61aeb93933 100644
> --- a/arch/arm64/kernel/entry-common.c
> +++ b/arch/arm64/kernel/entry-common.c
> @@ -14,6 +14,7 @@
> #include <linux/sched.h>
> #include <linux/sched/debug.h>
> #include <linux/thread_info.h>
> +#include <linux/entry-common.h>
>
> #include <asm/cpufeature.h>
> #include <asm/daifflags.h>
> @@ -28,201 +29,15 @@
> #include <asm/sysreg.h>
> #include <asm/system_misc.h>
>
> -/*
> - * Handle IRQ/context state management when entering from kernel mode.
> - * Before this function is called it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - *
> - * This is intended to match the logic in irqentry_enter(), handling the kernel
> - * mode transitions only.
> - */
> -static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)
> -{
> - regs->exit_rcu = false;
> -
> - if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
> - lockdep_hardirqs_off(CALLER_ADDR0);
> - ct_irq_enter();
> - trace_hardirqs_off_finish();
> -
> - regs->exit_rcu = true;
> - return;
> - }
> -
> - lockdep_hardirqs_off(CALLER_ADDR0);
> - rcu_irq_enter_check_tick();
> - trace_hardirqs_off_finish();
> -}
> -
> -static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
> -{
> - __enter_from_kernel_mode(regs);
> - mte_check_tfsr_entry();
> - mte_disable_tco_entry(current);
> -}
> -
> -/*
> - * Handle IRQ/context state management when exiting to kernel mode.
> - * After this function returns it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - *
> - * This is intended to match the logic in irqentry_exit(), handling the kernel
> - * mode transitions only, and with preemption handled elsewhere.
> - */
> -static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)
> -{
> - lockdep_assert_irqs_disabled();
> -
> - if (interrupts_enabled(regs)) {
> - if (regs->exit_rcu) {
> - trace_hardirqs_on_prepare();
> - lockdep_hardirqs_on_prepare();
> - ct_irq_exit();
> - lockdep_hardirqs_on(CALLER_ADDR0);
> - return;
> - }
> -
> - trace_hardirqs_on();
> - } else {
> - if (regs->exit_rcu)
> - ct_irq_exit();
> - }
> -}
> -
> -static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
> -{
> - mte_check_tfsr_exit();
> - __exit_to_kernel_mode(regs);
> -}
> -
> -/*
> - * Handle IRQ/context state management when entering from user mode.
> - * Before this function is called it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - */
> -static __always_inline void __enter_from_user_mode(void)
> +static __always_inline void exit_to_user_mode_wrapper(struct pt_regs *regs)
> {
> - lockdep_hardirqs_off(CALLER_ADDR0);
> - CT_WARN_ON(ct_state() != CONTEXT_USER);
> - user_exit_irqoff();
> - trace_hardirqs_off_finish();
> - mte_disable_tco_entry(current);
> -}
> -
> -static __always_inline void enter_from_user_mode(struct pt_regs *regs)
> -{
> - __enter_from_user_mode();
> -}
> -
> -/*
> - * Handle IRQ/context state management when exiting to user mode.
> - * After this function returns it is not safe to call regular kernel code,
> - * instrumentable code, or any code which may trigger an exception.
> - */
> -static __always_inline void __exit_to_user_mode(void)
> -{
> - trace_hardirqs_on_prepare();
> - lockdep_hardirqs_on_prepare();
> - user_enter_irqoff();
> - lockdep_hardirqs_on(CALLER_ADDR0);
> -}
> -
> -static void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags)
> -{
> - do {
> - local_irq_enable();
> -
> - if (thread_flags & _TIF_NEED_RESCHED)
> - schedule();
> -
> - if (thread_flags & _TIF_UPROBE)
> - uprobe_notify_resume(regs);
> -
> - if (thread_flags & _TIF_MTE_ASYNC_FAULT) {
> - clear_thread_flag(TIF_MTE_ASYNC_FAULT);
> - send_sig_fault(SIGSEGV, SEGV_MTEAERR,
> - (void __user *)NULL, current);
> - }
> -
> - if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
> - do_signal(regs);
> -
> - if (thread_flags & _TIF_NOTIFY_RESUME)
> - resume_user_mode_work(regs);
> -
> - if (thread_flags & _TIF_FOREIGN_FPSTATE)
> - fpsimd_restore_current_state();
> -
> - local_irq_disable();
> - thread_flags = read_thread_flags();
> - } while (thread_flags & _TIF_WORK_MASK);
> -}
> -
> -static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)
> -{
> - unsigned long flags;
> -
> local_irq_disable();
> -
> - flags = read_thread_flags();
> - if (unlikely(flags & _TIF_WORK_MASK))
> - do_notify_resume(regs, flags);
> -
> - local_daif_mask();
> -
> - lockdep_sys_exit();
> -}
> -
> -static __always_inline void exit_to_user_mode(struct pt_regs *regs)
> -{
> - exit_to_user_mode_prepare(regs);
> - mte_check_tfsr_exit();
> - __exit_to_user_mode();
> + irqentry_exit_to_user_mode(regs);
> }
>
> asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
> {
> - exit_to_user_mode(regs);
> -}
> -
> -/*
> - * Handle IRQ/context state management when entering an NMI from user/kernel
> - * mode. Before this function is called it is not safe to call regular kernel
> - * code, instrumentable code, or any code which may trigger an exception.
> - */
> -static void noinstr arm64_enter_nmi(struct pt_regs *regs)
> -{
> - regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
> -
> - __nmi_enter();
> - lockdep_hardirqs_off(CALLER_ADDR0);
> - lockdep_hardirq_enter();
> - ct_nmi_enter();
> -
> - trace_hardirqs_off_finish();
> - ftrace_nmi_enter();
> -}
> -
> -/*
> - * Handle IRQ/context state management when exiting an NMI from user/kernel
> - * mode. After this function returns it is not safe to call regular kernel
> - * code, instrumentable code, or any code which may trigger an exception.
> - */
> -static void noinstr arm64_exit_nmi(struct pt_regs *regs)
> -{
> - bool restore = regs->lockdep_hardirqs;
> -
> - ftrace_nmi_exit();
> - if (restore) {
> - trace_hardirqs_on_prepare();
> - lockdep_hardirqs_on_prepare();
> - }
> -
> - ct_nmi_exit();
> - lockdep_hardirq_exit();
> - if (restore)
> - lockdep_hardirqs_on(CALLER_ADDR0);
> - __nmi_exit();
> + exit_to_user_mode_wrapper(regs);
> }
>
> /*
> @@ -259,48 +74,6 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
> lockdep_hardirqs_on(CALLER_ADDR0);
> }
>
> -#ifdef CONFIG_PREEMPT_DYNAMIC
> -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
> -#define need_irq_preemption() \
> - (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
> -#else
> -#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION))
> -#endif
> -
> -static void __sched arm64_preempt_schedule_irq(void)
> -{
> - if (!need_irq_preemption())
> - return;
> -
> - /*
> - * Note: thread_info::preempt_count includes both thread_info::count
> - * and thread_info::need_resched, and is not equivalent to
> - * preempt_count().
> - */
> - if (READ_ONCE(current_thread_info()->preempt_count) != 0)
> - return;
> -
> - /*
> - * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
> - * priority masking is used the GIC irqchip driver will clear DAIF.IF
> - * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
> - * DAIF we must have handled an NMI, so skip preemption.
> - */
> - if (system_uses_irq_prio_masking() && read_sysreg(daif))
> - return;
> -
> - /*
> - * Preempting a task from an IRQ means we leave copies of PSTATE
> - * on the stack. cpufeature's enable calls may modify PSTATE, but
> - * resuming one of these preempted tasks would undo those changes.
> - *
> - * Only allow a task to be preempted once cpufeatures have been
> - * enabled.
> - */
> - if (system_capabilities_finalized())
> - preempt_schedule_irq();
> -}
> -
> static void do_interrupt_handler(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> {
> @@ -320,7 +93,7 @@ extern void (*handle_arch_fiq)(struct pt_regs *);
> static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
> unsigned long esr)
> {
> - arm64_enter_nmi(regs);
> + irqentry_nmi_enter(regs);
>
> console_verbose();
>
> @@ -426,41 +199,43 @@ UNHANDLED(el1t, 64, error)
> static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
> {
> unsigned long far = read_sysreg(far_el1);
> + irqentry_state_t state = irqentry_enter(regs);
>
> - enter_from_kernel_mode(regs);
> local_daif_inherit(regs);
> do_mem_abort(far, esr, regs);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
> {
> unsigned long far = read_sysreg(far_el1);
> + irqentry_state_t state = irqentry_enter(regs);
>
> - enter_from_kernel_mode(regs);
> local_daif_inherit(regs);
> do_sp_pc_abort(far, esr, regs);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_kernel_mode(regs);
> + irqentry_state_t state = irqentry_enter(regs);
> +
> local_daif_inherit(regs);
> do_el1_undef(regs, esr);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_kernel_mode(regs);
> + irqentry_state_t state = irqentry_enter(regs);
> +
> local_daif_inherit(regs);
> do_el1_bti(regs, esr);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
> @@ -475,11 +250,12 @@ static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
>
> static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_kernel_mode(regs);
> + irqentry_state_t state = irqentry_enter(regs);
> +
> local_daif_inherit(regs);
> do_el1_fpac(regs, esr);
> local_daif_mask();
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
>
> asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
> @@ -522,23 +298,22 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
> static __always_inline void __el1_pnmi(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> {
> - arm64_enter_nmi(regs);
> + irqentry_state_t state = irqentry_nmi_enter(regs);
> +
> do_interrupt_handler(regs, handler);
> - arm64_exit_nmi(regs);
> + irqentry_nmi_exit(regs, state);
> }
>
> static __always_inline void __el1_irq(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> {
> - enter_from_kernel_mode(regs);
> + irqentry_state_t state = irqentry_enter(regs);
>
> irq_enter_rcu();
> do_interrupt_handler(regs, handler);
> irq_exit_rcu();
>
> - arm64_preempt_schedule_irq();
> -
> - exit_to_kernel_mode(regs);
> + irqentry_exit(regs, state);
> }
> static void noinstr el1_interrupt(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> @@ -564,21 +339,22 @@ asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
> asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
> {
> unsigned long esr = read_sysreg(esr_el1);
> + irqentry_state_t state;
>
> local_daif_restore(DAIF_ERRCTX);
> - arm64_enter_nmi(regs);
> + state = irqentry_nmi_enter(regs);
> do_serror(regs, esr);
> - arm64_exit_nmi(regs);
> + irqentry_nmi_exit(regs, state);
> }
>
> static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
> {
> unsigned long far = read_sysreg(far_el1);
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_mem_abort(far, esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
> @@ -593,50 +369,50 @@ static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
> if (!is_ttbr0_addr(far))
> arm64_apply_bp_hardening();
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_mem_abort(far, esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_fpsimd_acc(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_sve_acc(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_sme_acc(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_fpsimd_exc(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_sys(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
> @@ -646,50 +422,50 @@ static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
> if (!is_ttbr0_addr(instruction_pointer(regs)))
> arm64_apply_bp_hardening();
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_sp_pc_abort(far, esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_sp_pc_abort(regs->sp, esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_undef(regs, esr);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_bti(struct pt_regs *regs)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_bti(regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_mops(regs, esr);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> bad_el0_sync(regs, 0, esr);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
> @@ -697,28 +473,28 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
> /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
> unsigned long far = read_sysreg(far_el1);
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> do_debug_exception(far, esr, regs);
> local_daif_restore(DAIF_PROCCTX);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_svc(struct pt_regs *regs)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> cortex_a76_erratum_1463225_svc_handler();
> fp_user_discard();
> local_daif_restore(DAIF_PROCCTX);
> do_el0_svc(regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_fpac(regs, esr);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
> @@ -783,7 +559,7 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
> static void noinstr el0_interrupt(struct pt_regs *regs,
> void (*handler)(struct pt_regs *))
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
>
> write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
>
> @@ -794,7 +570,7 @@ static void noinstr el0_interrupt(struct pt_regs *regs,
> do_interrupt_handler(regs, handler);
> irq_exit_rcu();
>
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
> @@ -820,14 +596,15 @@ asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
> static void noinstr __el0_error_handler_common(struct pt_regs *regs)
> {
> unsigned long esr = read_sysreg(esr_el1);
> + irqentry_state_t state_nmi;
>
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_ERRCTX);
> - arm64_enter_nmi(regs);
> + state_nmi = irqentry_nmi_enter(regs);
> do_serror(regs, esr);
> - arm64_exit_nmi(regs);
> + irqentry_nmi_exit(regs, state_nmi);
> local_daif_restore(DAIF_PROCCTX);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
> @@ -838,19 +615,19 @@ asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
> #ifdef CONFIG_COMPAT
> static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> local_daif_restore(DAIF_PROCCTX);
> do_el0_cp15(esr, regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> static void noinstr el0_svc_compat(struct pt_regs *regs)
> {
> - enter_from_user_mode(regs);
> + irqentry_enter_from_user_mode(regs);
> cortex_a76_erratum_1463225_svc_handler();
> local_daif_restore(DAIF_PROCCTX);
> do_el0_svc_compat(regs);
> - exit_to_user_mode(regs);
> + exit_to_user_mode_wrapper(regs);
> }
>
> asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
> @@ -924,7 +701,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
> unsigned long esr = read_sysreg(esr_el1);
> unsigned long far = read_sysreg(far_el1);
>
> - arm64_enter_nmi(regs);
> + irqentry_nmi_enter(regs);
> panic_bad_stack(regs, esr, far);
> }
> #endif /* CONFIG_VMAP_STACK */
> @@ -933,6 +710,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
> asmlinkage noinstr unsigned long
> __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
> {
> + irqentry_state_t state;
> unsigned long ret;
>
> /*
> @@ -957,9 +735,9 @@ __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
> else if (cpu_has_pan())
> set_pstate_pan(0);
>
> - arm64_enter_nmi(regs);
> + state = irqentry_nmi_enter(regs);
> ret = do_sdei_event(regs, arg);
> - arm64_exit_nmi(regs);
> + irqentry_nmi_exit(regs, state);
>
> return ret;
> }
> diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
> index 60fd85d5119d..9cc0f450fa76 100644
> --- a/arch/arm64/kernel/ptrace.c
> +++ b/arch/arm64/kernel/ptrace.c
> @@ -41,9 +41,6 @@
> #include <asm/traps.h>
> #include <asm/system_misc.h>
>
> -#define CREATE_TRACE_POINTS
> -#include <trace/events/syscalls.h>
> -
> struct pt_regs_offset {
> const char *name;
> int offset;
> @@ -2179,104 +2176,6 @@ long arch_ptrace(struct task_struct *child, long request,
> return ptrace_request(child, request, addr, data);
> }
>
> -enum ptrace_syscall_dir {
> - PTRACE_SYSCALL_ENTER = 0,
> - PTRACE_SYSCALL_EXIT,
> -};
> -
> -static void report_syscall_enter(struct pt_regs *regs)
> -{
> - int regno;
> - unsigned long saved_reg;
> -
> - /*
> - * We have some ABI weirdness here in the way that we handle syscall
> - * exit stops because we indicate whether or not the stop has been
> - * signalled from syscall entry or syscall exit by clobbering a general
> - * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
> - * and restoring its old value after the stop. This means that:
> - *
> - * - Any writes by the tracer to this register during the stop are
> - * ignored/discarded.
> - *
> - * - The actual value of the register is not available during the stop,
> - * so the tracer cannot save it and restore it later.
> - *
> - * - Syscall stops behave differently to seccomp and pseudo-step traps
> - * (the latter do not nobble any registers).
> - */
> - regno = (is_compat_task() ? 12 : 7);
> - saved_reg = regs->regs[regno];
> - regs->regs[regno] = PTRACE_SYSCALL_ENTER;
> -
> - if (ptrace_report_syscall_entry(regs))
> - forget_syscall(regs);
> - regs->regs[regno] = saved_reg;
> -}
> -
> -static void report_syscall_exit(struct pt_regs *regs)
> -{
> - int regno;
> - unsigned long saved_reg;
> -
> - /* See comment for report_syscall_enter() */
> - regno = (is_compat_task() ? 12 : 7);
> - saved_reg = regs->regs[regno];
> - regs->regs[regno] = PTRACE_SYSCALL_EXIT;
> -
> - if (!test_thread_flag(TIF_SINGLESTEP)) {
> - ptrace_report_syscall_exit(regs, 0);
> - regs->regs[regno] = saved_reg;
> - } else {
> - regs->regs[regno] = saved_reg;
> -
> - /*
> - * Signal a pseudo-step exception since we are stepping but
> - * tracer modifications to the registers may have rewound the
> - * state machine.
> - */
> - ptrace_report_syscall_exit(regs, 1);
> - }
> -}
> -
> -int syscall_trace_enter(struct pt_regs *regs)
> -{
> - unsigned long flags = read_thread_flags();
> -
> - if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
> - report_syscall_enter(regs);
> - if (flags & _TIF_SYSCALL_EMU)
> - return NO_SYSCALL;
> - }
> -
> - /* Do the secure computing after ptrace; failures should be fast. */
> - if (secure_computing() == -1)
> - return NO_SYSCALL;
> -
> - if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
> - trace_sys_enter(regs, regs->syscallno);
> -
> - audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
> - regs->regs[2], regs->regs[3]);
> -
> - return regs->syscallno;
> -}
> -
> -void syscall_trace_exit(struct pt_regs *regs)
> -{
> - unsigned long flags = read_thread_flags();
> -
> - audit_syscall_exit(regs);
> -
> - if (flags & _TIF_SYSCALL_TRACEPOINT)
> - trace_sys_exit(regs, syscall_get_return_value(current, regs));
> -
> - if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP))
> - report_syscall_exit(regs);
> -
> - rseq_syscall(regs);
> -}
> -
> /*
> * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
> * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
> diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
> index 4a77f4976e11..2982f6db6d96 100644
> --- a/arch/arm64/kernel/signal.c
> +++ b/arch/arm64/kernel/signal.c
> @@ -19,6 +19,7 @@
> #include <linux/ratelimit.h>
> #include <linux/rseq.h>
> #include <linux/syscalls.h>
> +#include <linux/entry-common.h>
>
> #include <asm/daifflags.h>
> #include <asm/debug-monitors.h>
> @@ -1266,7 +1267,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
> * the kernel can handle, and then we build all the user-level signal handling
> * stack-frames in one go after that.
> */
> -void do_signal(struct pt_regs *regs)
> +void arch_do_signal_or_restart(struct pt_regs *regs)
> {
> unsigned long continue_addr = 0, restart_addr = 0;
> int retval = 0;
> diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c
> index ad198262b981..160ac9d15c27 100644
> --- a/arch/arm64/kernel/syscall.c
> +++ b/arch/arm64/kernel/syscall.c
> @@ -7,6 +7,7 @@
> #include <linux/ptrace.h>
> #include <linux/randomize_kstack.h>
> #include <linux/syscalls.h>
> +#include <linux/entry-common.h>
>
> #include <asm/debug-monitors.h>
> #include <asm/exception.h>
> @@ -66,14 +67,15 @@ static void invoke_syscall(struct pt_regs *regs, unsigned int scno,
> choose_random_kstack_offset(get_random_u16() & 0x1FF);
> }
>
> -static inline bool has_syscall_work(unsigned long flags)
> +static inline bool has_syscall_work(unsigned long work)
> {
> - return unlikely(flags & _TIF_SYSCALL_WORK);
> + return unlikely(work & SYSCALL_WORK_ENTER);
> }
>
> static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> const syscall_fn_t syscall_table[])
> {
> + unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
> unsigned long flags = read_thread_flags();
>
> regs->orig_x0 = regs->regs[0];
> @@ -107,7 +109,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> return;
> }
>
> - if (has_syscall_work(flags)) {
> + if (has_syscall_work(work)) {
> /*
> * The de-facto standard way to skip a system call using ptrace
> * is to set the system call to -1 (NO_SYSCALL) and set x0 to a
> @@ -125,7 +127,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> */
> if (scno == NO_SYSCALL)
> syscall_set_return_value(current, regs, -ENOSYS, 0);
> - scno = syscall_trace_enter(regs);
> + scno = syscall_trace_enter(regs, regs->syscallno, work);
> if (scno == NO_SYSCALL)
> goto trace_exit;
> }
> @@ -137,14 +139,14 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
> * check again. However, if we were tracing entry, then we always trace
> * exit regardless, as the old entry assembly did.
> */
> - if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
> - flags = read_thread_flags();
> - if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
> + if (!has_syscall_work(work) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
> + work = READ_ONCE(current_thread_info()->syscall_work);
> + if (!has_syscall_work(work) && !report_single_step(work))
> return;
> }
>
> trace_exit:
> - syscall_trace_exit(regs);
> + syscall_exit_work(regs, work);
> }
>
> void do_el0_svc(struct pt_regs *regs)
© 2016 - 2025 Red Hat, Inc.