The irqsoff tracer is rarely enabled in production systems due to the
non-negligible overhead it introduces—even when unused. This is caused
by how trace_hardirqs_on/off() are always invoked in
local_irq_enable/disable(), evaluate the tracepoint static key.
This patch reduces the overhead in the common case where the tracepoint
is disabled by performing the static key check earlier, avoiding the
call to trace_hardirqs_on/off() entirely.
This makes the impact of disabled preemptirq IRQ tracing negligible in
performance-sensitive environments.
Signed-off-by: Wander Lairson Costa <wander@redhat.com>
Suggested-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Clark Williams <williams@redhat.com>
Cc: Gabriele Monaco <gmonaco@redhat.com>
Cc: Juri Lelli <juri.lelli@redhat.com>
---
include/linux/irqflags.h | 30 +++++++++++++++++++++---------
kernel/trace/trace_preemptirq.c | 3 +++
2 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h
index 57b074e0cfbb..40e456fa3d10 100644
--- a/include/linux/irqflags.h
+++ b/include/linux/irqflags.h
@@ -17,6 +17,7 @@
#include <linux/cleanup.h>
#include <asm/irqflags.h>
#include <asm/percpu.h>
+#include <linux/tracepoint-defs.h>
struct task_struct;
@@ -197,9 +198,17 @@ extern void warn_bogus_irq_restore(void);
*/
#ifdef CONFIG_TRACE_IRQFLAGS
+DECLARE_TRACEPOINT(irq_enable);
+DECLARE_TRACEPOINT(irq_disable);
+
+#define __trace_enabled(tp) \
+ (IS_ENABLED(CONFIG_PROVE_LOCKING) || \
+ tracepoint_enabled(tp))
+
#define local_irq_enable() \
do { \
- trace_hardirqs_on(); \
+ if (__trace_enabled(irq_enable)) \
+ trace_hardirqs_on(); \
raw_local_irq_enable(); \
} while (0)
@@ -207,31 +216,34 @@ extern void warn_bogus_irq_restore(void);
do { \
bool was_disabled = raw_irqs_disabled();\
raw_local_irq_disable(); \
- if (!was_disabled) \
+ if (__trace_enabled(irq_disable) && \
+ !was_disabled) \
trace_hardirqs_off(); \
} while (0)
#define local_irq_save(flags) \
do { \
raw_local_irq_save(flags); \
- if (!raw_irqs_disabled_flags(flags)) \
+ if (__trace_enabled(irq_disable) && \
+ !raw_irqs_disabled_flags(flags)) \
trace_hardirqs_off(); \
} while (0)
#define local_irq_restore(flags) \
do { \
- if (!raw_irqs_disabled_flags(flags)) \
+ if (__trace_enabled(irq_enable) && \
+ !raw_irqs_disabled_flags(flags)) \
trace_hardirqs_on(); \
raw_local_irq_restore(flags); \
} while (0)
-#define safe_halt() \
- do { \
- trace_hardirqs_on(); \
- raw_safe_halt(); \
+#define safe_halt() \
+ do { \
+ if (__trace_enabled(irq_enable)) \
+ trace_hardirqs_on(); \
+ raw_safe_halt(); \
} while (0)
-
#else /* !CONFIG_TRACE_IRQFLAGS */
#define local_irq_enable() do { raw_local_irq_enable(); } while (0)
diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c
index 0c42b15c3800..90ee65db4516 100644
--- a/kernel/trace/trace_preemptirq.c
+++ b/kernel/trace/trace_preemptirq.c
@@ -111,6 +111,9 @@ void trace_hardirqs_off(void)
}
EXPORT_SYMBOL(trace_hardirqs_off);
NOKPROBE_SYMBOL(trace_hardirqs_off);
+
+EXPORT_TRACEPOINT_SYMBOL(irq_disable);
+EXPORT_TRACEPOINT_SYMBOL(irq_enable);
#endif /* CONFIG_TRACE_IRQFLAGS */
#ifdef CONFIG_TRACE_PREEMPT_TOGGLE
--
2.50.0
Hi Wander,
kernel test robot noticed the following build errors:
[auto build test ERROR on trace/for-next]
[also build test ERROR on tip/sched/core linus/master v6.16-rc4 next-20250701]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Wander-Lairson-Costa/trace-preemptirq-reduce-overhead-of-irq_enable-disable-tracepoints/20250701-035446
base: https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace for-next
patch link: https://lore.kernel.org/r/20250630195243.701516-2-wander%40redhat.com
patch subject: [PATCH v2 1/2] trace/preemptirq: reduce overhead of irq_enable/disable tracepoints
config: arm-randconfig-004-20250701 (https://download.01.org/0day-ci/archive/20250701/202507011949.4wad2fIh-lkp@intel.com/config)
compiler: clang version 21.0.0git (https://github.com/llvm/llvm-project e04c938cc08a90ae60440ce22d072ebc69d67ee8)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250701/202507011949.4wad2fIh-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202507011949.4wad2fIh-lkp@intel.com/
All errors (new ones prefixed by >>):
In file included from kernel/bounds.c:13:
In file included from include/linux/log2.h:12:
In file included from include/linux/bitops.h:68:
In file included from arch/arm/include/asm/bitops.h:28:
In file included from include/linux/irqflags.h:20:
In file included from include/linux/tracepoint-defs.h:11:
In file included from include/linux/atomic.h:7:
In file included from arch/arm/include/asm/atomic.h:16:
In file included from arch/arm/include/asm/cmpxchg.h:124:
include/asm-generic/cmpxchg-local.h:26:2: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
26 | raw_local_irq_save(flags);
| ^
include/asm-generic/cmpxchg-local.h:26:2: note: did you mean 'arch_local_irq_save'?
arch/arm/include/asm/irqflags.h:72:29: note: 'arch_local_irq_save' declared here
72 | static inline unsigned long arch_local_irq_save(void)
| ^
arch/arm/include/asm/irqflags.h:71:29: note: expanded from macro 'arch_local_irq_save'
71 | #define arch_local_irq_save arch_local_irq_save
| ^
In file included from kernel/bounds.c:13:
In file included from include/linux/log2.h:12:
In file included from include/linux/bitops.h:68:
In file included from arch/arm/include/asm/bitops.h:28:
In file included from include/linux/irqflags.h:20:
In file included from include/linux/tracepoint-defs.h:11:
In file included from include/linux/atomic.h:7:
In file included from arch/arm/include/asm/atomic.h:16:
In file included from arch/arm/include/asm/cmpxchg.h:124:
include/asm-generic/cmpxchg-local.h:47:2: error: call to undeclared function 'raw_local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
47 | raw_local_irq_restore(flags);
| ^
include/asm-generic/cmpxchg-local.h:47:2: note: did you mean 'arch_local_irq_restore'?
arch/arm/include/asm/irqflags.h:169:20: note: 'arch_local_irq_restore' declared here
169 | static inline void arch_local_irq_restore(unsigned long flags)
| ^
arch/arm/include/asm/irqflags.h:168:32: note: expanded from macro 'arch_local_irq_restore'
168 | #define arch_local_irq_restore arch_local_irq_restore
| ^
In file included from kernel/bounds.c:13:
In file included from include/linux/log2.h:12:
In file included from include/linux/bitops.h:68:
In file included from arch/arm/include/asm/bitops.h:28:
In file included from include/linux/irqflags.h:20:
In file included from include/linux/tracepoint-defs.h:11:
In file included from include/linux/atomic.h:7:
In file included from arch/arm/include/asm/atomic.h:16:
In file included from arch/arm/include/asm/cmpxchg.h:124:
include/asm-generic/cmpxchg-local.h:60:2: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
60 | raw_local_irq_save(flags);
| ^
include/asm-generic/cmpxchg-local.h:64:2: error: call to undeclared function 'raw_local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
64 | raw_local_irq_restore(flags);
| ^
In file included from kernel/bounds.c:13:
In file included from include/linux/log2.h:12:
In file included from include/linux/bitops.h:68:
In file included from arch/arm/include/asm/bitops.h:28:
In file included from include/linux/irqflags.h:20:
In file included from include/linux/tracepoint-defs.h:11:
In file included from include/linux/atomic.h:7:
In file included from arch/arm/include/asm/atomic.h:16:
In file included from arch/arm/include/asm/cmpxchg.h:148:
>> include/asm-generic/cmpxchg.h:33:3: error: call to undeclared function 'local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
33 | local_irq_save(flags);
| ^
>> include/asm-generic/cmpxchg.h:36:3: error: call to undeclared function 'local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
36 | local_irq_restore(flags);
| ^
In file included from kernel/bounds.c:13:
In file included from include/linux/log2.h:12:
In file included from include/linux/bitops.h:68:
In file included from arch/arm/include/asm/bitops.h:28:
In file included from include/linux/irqflags.h:20:
In file included from include/linux/tracepoint-defs.h:11:
In file included from include/linux/atomic.h:7:
>> arch/arm/include/asm/atomic.h:215:2: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
215 | raw_local_irq_save(flags);
| ^
>> arch/arm/include/asm/atomic.h:219:2: error: call to undeclared function 'raw_local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
219 | raw_local_irq_restore(flags);
| ^
arch/arm/include/asm/atomic.h:232:1: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
232 | ATOMIC_OPS(add, +=, add)
| ^
arch/arm/include/asm/atomic.h:228:2: note: expanded from macro 'ATOMIC_OPS'
228 | ATOMIC_OP(op, c_op, asm_op) \
| ^
arch/arm/include/asm/atomic.h:167:2: note: expanded from macro 'ATOMIC_OP'
167 | raw_local_irq_save(flags); \
| ^
arch/arm/include/asm/atomic.h:232:1: error: call to undeclared function 'raw_local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:228:2: note: expanded from macro 'ATOMIC_OPS'
228 | ATOMIC_OP(op, c_op, asm_op) \
| ^
arch/arm/include/asm/atomic.h:169:2: note: expanded from macro 'ATOMIC_OP'
169 | raw_local_irq_restore(flags); \
| ^
arch/arm/include/asm/atomic.h:232:1: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:229:2: note: expanded from macro 'ATOMIC_OPS'
229 | ATOMIC_OP_RETURN(op, c_op, asm_op) \
| ^
arch/arm/include/asm/atomic.h:178:2: note: expanded from macro 'ATOMIC_OP_RETURN'
178 | raw_local_irq_save(flags); \
| ^
arch/arm/include/asm/atomic.h:232:1: error: call to undeclared function 'raw_local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:229:2: note: expanded from macro 'ATOMIC_OPS'
229 | ATOMIC_OP_RETURN(op, c_op, asm_op) \
| ^
arch/arm/include/asm/atomic.h:181:2: note: expanded from macro 'ATOMIC_OP_RETURN'
181 | raw_local_irq_restore(flags); \
| ^
arch/arm/include/asm/atomic.h:232:1: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:230:2: note: expanded from macro 'ATOMIC_OPS'
230 | ATOMIC_FETCH_OP(op, c_op, asm_op)
| ^
arch/arm/include/asm/atomic.h:192:2: note: expanded from macro 'ATOMIC_FETCH_OP'
192 | raw_local_irq_save(flags); \
| ^
arch/arm/include/asm/atomic.h:232:1: error: call to undeclared function 'raw_local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:230:2: note: expanded from macro 'ATOMIC_OPS'
230 | ATOMIC_FETCH_OP(op, c_op, asm_op)
| ^
arch/arm/include/asm/atomic.h:195:2: note: expanded from macro 'ATOMIC_FETCH_OP'
195 | raw_local_irq_restore(flags); \
| ^
arch/arm/include/asm/atomic.h:233:1: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
233 | ATOMIC_OPS(sub, -=, sub)
| ^
arch/arm/include/asm/atomic.h:228:2: note: expanded from macro 'ATOMIC_OPS'
228 | ATOMIC_OP(op, c_op, asm_op) \
| ^
arch/arm/include/asm/atomic.h:167:2: note: expanded from macro 'ATOMIC_OP'
167 | raw_local_irq_save(flags); \
| ^
arch/arm/include/asm/atomic.h:233:1: error: call to undeclared function 'raw_local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:228:2: note: expanded from macro 'ATOMIC_OPS'
228 | ATOMIC_OP(op, c_op, asm_op) \
| ^
arch/arm/include/asm/atomic.h:169:2: note: expanded from macro 'ATOMIC_OP'
169 | raw_local_irq_restore(flags); \
| ^
arch/arm/include/asm/atomic.h:233:1: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:229:2: note: expanded from macro 'ATOMIC_OPS'
229 | ATOMIC_OP_RETURN(op, c_op, asm_op) \
| ^
arch/arm/include/asm/atomic.h:178:2: note: expanded from macro 'ATOMIC_OP_RETURN'
178 | raw_local_irq_save(flags); \
| ^
arch/arm/include/asm/atomic.h:233:1: error: call to undeclared function 'raw_local_irq_restore'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:229:2: note: expanded from macro 'ATOMIC_OPS'
229 | ATOMIC_OP_RETURN(op, c_op, asm_op) \
| ^
arch/arm/include/asm/atomic.h:181:2: note: expanded from macro 'ATOMIC_OP_RETURN'
181 | raw_local_irq_restore(flags); \
| ^
arch/arm/include/asm/atomic.h:233:1: error: call to undeclared function 'raw_local_irq_save'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
arch/arm/include/asm/atomic.h:230:2: note: expanded from macro 'ATOMIC_OPS'
230 | ATOMIC_FETCH_OP(op, c_op, asm_op)
| ^
arch/arm/include/asm/atomic.h:192:2: note: expanded from macro 'ATOMIC_FETCH_OP'
192 | raw_local_irq_save(flags); \
| ^
fatal error: too many errors emitted, stopping now [-ferror-limit=]
20 errors generated.
make[3]: *** [scripts/Makefile.build:98: kernel/bounds.s] Error 1 shuffle=3679043308
make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1274: prepare0] Error 2 shuffle=3679043308
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:248: __sub-make] Error 2 shuffle=3679043308
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:248: __sub-make] Error 2 shuffle=3679043308
make: Target 'prepare' not remade because of errors.
vim +/local_irq_save +33 include/asm-generic/cmpxchg.h
b4816afa398670 David Howells 2012-03-28 22
b4816afa398670 David Howells 2012-03-28 23 static inline
82b993e8249ae3 Mark Rutland 2021-05-25 24 unsigned long __generic_xchg(unsigned long x, volatile void *ptr, int size)
b4816afa398670 David Howells 2012-03-28 25 {
b4816afa398670 David Howells 2012-03-28 26 unsigned long ret, flags;
b4816afa398670 David Howells 2012-03-28 27
b4816afa398670 David Howells 2012-03-28 28 switch (size) {
b4816afa398670 David Howells 2012-03-28 29 case 1:
b4816afa398670 David Howells 2012-03-28 30 #ifdef __xchg_u8
b4816afa398670 David Howells 2012-03-28 31 return __xchg_u8(x, ptr);
b4816afa398670 David Howells 2012-03-28 32 #else
b4816afa398670 David Howells 2012-03-28 @33 local_irq_save(flags);
b4816afa398670 David Howells 2012-03-28 34 ret = *(volatile u8 *)ptr;
656e9007ef5862 Arnd Bergmann 2023-03-02 35 *(volatile u8 *)ptr = (x & 0xffu);
b4816afa398670 David Howells 2012-03-28 @36 local_irq_restore(flags);
b4816afa398670 David Howells 2012-03-28 37 return ret;
b4816afa398670 David Howells 2012-03-28 38 #endif /* __xchg_u8 */
b4816afa398670 David Howells 2012-03-28 39
b4816afa398670 David Howells 2012-03-28 40 case 2:
b4816afa398670 David Howells 2012-03-28 41 #ifdef __xchg_u16
b4816afa398670 David Howells 2012-03-28 42 return __xchg_u16(x, ptr);
b4816afa398670 David Howells 2012-03-28 43 #else
b4816afa398670 David Howells 2012-03-28 44 local_irq_save(flags);
b4816afa398670 David Howells 2012-03-28 45 ret = *(volatile u16 *)ptr;
656e9007ef5862 Arnd Bergmann 2023-03-02 46 *(volatile u16 *)ptr = (x & 0xffffu);
b4816afa398670 David Howells 2012-03-28 47 local_irq_restore(flags);
b4816afa398670 David Howells 2012-03-28 48 return ret;
b4816afa398670 David Howells 2012-03-28 49 #endif /* __xchg_u16 */
b4816afa398670 David Howells 2012-03-28 50
b4816afa398670 David Howells 2012-03-28 51 case 4:
b4816afa398670 David Howells 2012-03-28 52 #ifdef __xchg_u32
b4816afa398670 David Howells 2012-03-28 53 return __xchg_u32(x, ptr);
b4816afa398670 David Howells 2012-03-28 54 #else
b4816afa398670 David Howells 2012-03-28 55 local_irq_save(flags);
b4816afa398670 David Howells 2012-03-28 56 ret = *(volatile u32 *)ptr;
656e9007ef5862 Arnd Bergmann 2023-03-02 57 *(volatile u32 *)ptr = (x & 0xffffffffu);
b4816afa398670 David Howells 2012-03-28 58 local_irq_restore(flags);
b4816afa398670 David Howells 2012-03-28 59 return ret;
b4816afa398670 David Howells 2012-03-28 60 #endif /* __xchg_u32 */
b4816afa398670 David Howells 2012-03-28 61
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
© 2016 - 2025 Red Hat, Inc.