Architectures using the generic entry code can be optimized by having
syscall_exit_to_user_mode inlined.
Signed-off-by: Charlie Jenkins <charlie@rivosinc.com>
---
include/linux/entry-common.h | 43 ++++++++++++++++++++++++++++++++++++--
kernel/entry/common.c | 49 +-------------------------------------------
2 files changed, 42 insertions(+), 50 deletions(-)
diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h
index fc61d0205c97084acc89c8e45e088946f5e6d9b2..a46861ffd6858fadf4014c387e8f2f216a879c25 100644
--- a/include/linux/entry-common.h
+++ b/include/linux/entry-common.h
@@ -14,6 +14,7 @@
#include <linux/kmsan.h>
#include <asm/entry-common.h>
+#include <asm/syscall.h>
/*
* Define dummy _TIF work flags if not defined by the architecture or for
@@ -366,6 +367,15 @@ static __always_inline void exit_to_user_mode(void)
lockdep_hardirqs_on(CALLER_ADDR0);
}
+/**
+ * syscall_exit_work - Handle work before returning to user mode
+ * @regs: Pointer to current pt_regs
+ * @work: Current thread syscall work
+ *
+ * Do one-time syscall specific work.
+ */
+void syscall_exit_work(struct pt_regs *regs, unsigned long work);
+
/**
* syscall_exit_to_user_mode_work - Handle work before returning to user mode
* @regs: Pointer to currents pt_regs
@@ -379,7 +389,30 @@ static __always_inline void exit_to_user_mode(void)
* make the final state transitions. Interrupts must stay disabled between
* return from this function and the invocation of exit_to_user_mode().
*/
-void syscall_exit_to_user_mode_work(struct pt_regs *regs);
+static __always_inline void syscall_exit_to_user_mode_work(struct pt_regs *regs)
+{
+ unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
+ unsigned long nr = syscall_get_nr(current, regs);
+
+ CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
+
+ if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
+ if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
+ local_irq_enable();
+ }
+
+ rseq_syscall(regs);
+
+ /*
+ * Do one-time syscall specific work. If these work items are
+ * enabled, we want to run them exactly once per syscall exit with
+ * interrupts enabled.
+ */
+ if (unlikely(work & SYSCALL_WORK_EXIT))
+ syscall_exit_work(regs, work);
+ local_irq_disable_exit_to_user();
+ exit_to_user_mode_prepare(regs);
+}
/**
* syscall_exit_to_user_mode - Handle work before returning to user mode
@@ -410,7 +443,13 @@ void syscall_exit_to_user_mode_work(struct pt_regs *regs);
* exit_to_user_mode(). This function is preferred unless there is a
* compelling architectural reason to use the separate functions.
*/
-void syscall_exit_to_user_mode(struct pt_regs *regs);
+static __always_inline void syscall_exit_to_user_mode(struct pt_regs *regs)
+{
+ instrumentation_begin();
+ syscall_exit_to_user_mode_work(regs);
+ instrumentation_end();
+ exit_to_user_mode();
+}
/**
* irqentry_enter_from_user_mode - Establish state before invoking the irq handler
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index e33691d5adf7aab4af54cf2bf8e5ef5bd6ad1424..f55e421fb196dd5f9d4e34dd85ae096c774cf879 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -146,7 +146,7 @@ static inline bool report_single_step(unsigned long work)
return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP;
}
-static void syscall_exit_work(struct pt_regs *regs, unsigned long work)
+void syscall_exit_work(struct pt_regs *regs, unsigned long work)
{
bool step;
@@ -173,53 +173,6 @@ static void syscall_exit_work(struct pt_regs *regs, unsigned long work)
ptrace_report_syscall_exit(regs, step);
}
-/*
- * Syscall specific exit to user mode preparation. Runs with interrupts
- * enabled.
- */
-static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
-{
- unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
- unsigned long nr = syscall_get_nr(current, regs);
-
- CT_WARN_ON(ct_state() != CT_STATE_KERNEL);
-
- if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
- if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
- local_irq_enable();
- }
-
- rseq_syscall(regs);
-
- /*
- * Do one-time syscall specific work. If these work items are
- * enabled, we want to run them exactly once per syscall exit with
- * interrupts enabled.
- */
- if (unlikely(work & SYSCALL_WORK_EXIT))
- syscall_exit_work(regs, work);
-}
-
-static __always_inline void __syscall_exit_to_user_mode_work(struct pt_regs *regs)
-{
- syscall_exit_to_user_mode_prepare(regs);
- local_irq_disable_exit_to_user();
- exit_to_user_mode_prepare(regs);
-}
-
-void syscall_exit_to_user_mode_work(struct pt_regs *regs)
-{
- __syscall_exit_to_user_mode_work(regs);
-}
-
-__visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)
-{
- instrumentation_begin();
- __syscall_exit_to_user_mode_work(regs);
- instrumentation_end();
- exit_to_user_mode();
-}
-
noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
{
enter_from_user_mode(regs);
--
2.43.0Hi Charlie,
kernel test robot noticed the following build errors:
[auto build test ERROR on ffd294d346d185b70e28b1a28abe367bbfe53c04]
url: https://github.com/intel-lab-lkp/linux/commits/Charlie-Jenkins/riscv-entry-Convert-ret_from_fork-to-C/20250123-065957
base: ffd294d346d185b70e28b1a28abe367bbfe53c04
patch link: https://lore.kernel.org/r/20250122-riscv_optimize_entry-v1-4-4ee95559cfd0%40rivosinc.com
patch subject: [PATCH 4/4] entry: Inline syscall_exit_to_user_mode()
config: x86_64-randconfig-072-20250123 (https://download.01.org/0day-ci/archive/20250124/202501240101.mSdGpEoH-lkp@intel.com/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250124/202501240101.mSdGpEoH-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202501240101.mSdGpEoH-lkp@intel.com/
All errors (new ones prefixed by >>):
In file included from arch/x86/include/asm/bug.h:99,
from include/linux/bug.h:5,
from include/linux/thread_info.h:13,
from include/linux/sched.h:14,
from arch/x86/mm/fault.c:7:
include/linux/entry-common.h: In function 'syscall_exit_to_user_mode_work':
>> include/linux/entry-common.h:397:34: error: 'CONTEXT_KERNEL' undeclared (first use in this function); did you mean 'PERF_CONTEXT_KERNEL'?
397 | CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
| ^~~~~~~~~~~~~~
include/asm-generic/bug.h:123:32: note: in definition of macro 'WARN_ON'
123 | int __ret_warn_on = !!(condition); \
| ^~~~~~~~~
include/linux/entry-common.h:397:9: note: in expansion of macro 'CT_WARN_ON'
397 | CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
| ^~~~~~~~~~
include/linux/entry-common.h:397:34: note: each undeclared identifier is reported only once for each function it appears in
397 | CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
| ^~~~~~~~~~~~~~
include/asm-generic/bug.h:123:32: note: in definition of macro 'WARN_ON'
123 | int __ret_warn_on = !!(condition); \
| ^~~~~~~~~
include/linux/entry-common.h:397:9: note: in expansion of macro 'CT_WARN_ON'
397 | CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
| ^~~~~~~~~~
vim +397 include/linux/entry-common.h
378
379 /**
380 * syscall_exit_to_user_mode_work - Handle work before returning to user mode
381 * @regs: Pointer to currents pt_regs
382 *
383 * Same as step 1 and 2 of syscall_exit_to_user_mode() but without calling
384 * exit_to_user_mode() to perform the final transition to user mode.
385 *
386 * Calling convention is the same as for syscall_exit_to_user_mode() and it
387 * returns with all work handled and interrupts disabled. The caller must
388 * invoke exit_to_user_mode() before actually switching to user mode to
389 * make the final state transitions. Interrupts must stay disabled between
390 * return from this function and the invocation of exit_to_user_mode().
391 */
392 static __always_inline void syscall_exit_to_user_mode_work(struct pt_regs *regs)
393 {
394 unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
395 unsigned long nr = syscall_get_nr(current, regs);
396
> 397 CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
398
399 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
400 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
401 local_irq_enable();
402 }
403
404 rseq_syscall(regs);
405
406 /*
407 * Do one-time syscall specific work. If these work items are
408 * enabled, we want to run them exactly once per syscall exit with
409 * interrupts enabled.
410 */
411 if (unlikely(work & SYSCALL_WORK_EXIT))
412 syscall_exit_work(regs, work);
413 local_irq_disable_exit_to_user();
414 exit_to_user_mode_prepare(regs);
415 }
416
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Charlie,
kernel test robot noticed the following build errors:
[auto build test ERROR on ffd294d346d185b70e28b1a28abe367bbfe53c04]
url: https://github.com/intel-lab-lkp/linux/commits/Charlie-Jenkins/riscv-entry-Convert-ret_from_fork-to-C/20250123-065957
base: ffd294d346d185b70e28b1a28abe367bbfe53c04
patch link: https://lore.kernel.org/r/20250122-riscv_optimize_entry-v1-4-4ee95559cfd0%40rivosinc.com
patch subject: [PATCH 4/4] entry: Inline syscall_exit_to_user_mode()
config: x86_64-randconfig-071-20250123 (https://download.01.org/0day-ci/archive/20250124/202501240036.hxoscXJM-lkp@intel.com/config)
compiler: clang version 19.1.3 (https://github.com/llvm/llvm-project ab51eccf88f5321e7c60591c5546b254b6afab99)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250124/202501240036.hxoscXJM-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202501240036.hxoscXJM-lkp@intel.com/
All errors (new ones prefixed by >>):
In file included from arch/x86/kernel/ptrace.c:40:
In file included from arch/x86/include/asm/traps.h:9:
In file included from arch/x86/include/asm/idtentry.h:11:
>> include/linux/entry-common.h:397:27: error: use of undeclared identifier 'CONTEXT_KERNEL'
397 | CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
| ^
1 error generated.
vim +/CONTEXT_KERNEL +397 include/linux/entry-common.h
378
379 /**
380 * syscall_exit_to_user_mode_work - Handle work before returning to user mode
381 * @regs: Pointer to currents pt_regs
382 *
383 * Same as step 1 and 2 of syscall_exit_to_user_mode() but without calling
384 * exit_to_user_mode() to perform the final transition to user mode.
385 *
386 * Calling convention is the same as for syscall_exit_to_user_mode() and it
387 * returns with all work handled and interrupts disabled. The caller must
388 * invoke exit_to_user_mode() before actually switching to user mode to
389 * make the final state transitions. Interrupts must stay disabled between
390 * return from this function and the invocation of exit_to_user_mode().
391 */
392 static __always_inline void syscall_exit_to_user_mode_work(struct pt_regs *regs)
393 {
394 unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
395 unsigned long nr = syscall_get_nr(current, regs);
396
> 397 CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
398
399 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
400 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
401 local_irq_enable();
402 }
403
404 rseq_syscall(regs);
405
406 /*
407 * Do one-time syscall specific work. If these work items are
408 * enabled, we want to run them exactly once per syscall exit with
409 * interrupts enabled.
410 */
411 if (unlikely(work & SYSCALL_WORK_EXIT))
412 syscall_exit_work(regs, work);
413 local_irq_disable_exit_to_user();
414 exit_to_user_mode_prepare(regs);
415 }
416
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
© 2016 - 2025 Red Hat, Inc.