The irqsoff tracer is rarely enabled in production systems due to the
non-negligible overhead it introduces—even when unused. This is caused
by how trace_hardirqs_on/off() are always invoked in
local_irq_enable/disable(), evaluate the tracepoint static key.
This patch reduces the overhead in the common case where the tracepoint
is disabled by performing the static key check earlier, avoiding the
call to trace_hardirqs_on/off() entirely.
This makes the impact of disabled preemptirq IRQ tracing negligible in
performance-sensitive environments.
Signed-off-by: Wander Lairson Costa <wander@redhat.com>
Suggested-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Clark Williams <williams@redhat.com>
Cc: Gabriele Monaco <gmonaco@redhat.com>
Cc: Juri Lelli <juri.lelli@redhat.com>
---
include/linux/irqflags.h | 25 +++++++++++++++++--------
kernel/trace/trace_preemptirq.c | 3 +++
2 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h
index 57b074e0cfbb..54f931db7e3b 100644
--- a/include/linux/irqflags.h
+++ b/include/linux/irqflags.h
@@ -17,6 +17,7 @@
#include <linux/cleanup.h>
#include <asm/irqflags.h>
#include <asm/percpu.h>
+#include <linux/tracepoint-defs.h>
struct task_struct;
@@ -197,9 +198,13 @@ extern void warn_bogus_irq_restore(void);
*/
#ifdef CONFIG_TRACE_IRQFLAGS
+DECLARE_TRACEPOINT(irq_enable);
+DECLARE_TRACEPOINT(irq_disable);
+
#define local_irq_enable() \
do { \
- trace_hardirqs_on(); \
+ if (tracepoint_enabled(irq_enable)) \
+ trace_hardirqs_on(); \
raw_local_irq_enable(); \
} while (0)
@@ -207,28 +212,32 @@ extern void warn_bogus_irq_restore(void);
do { \
bool was_disabled = raw_irqs_disabled();\
raw_local_irq_disable(); \
- if (!was_disabled) \
+ if (tracepoint_enabled(irq_disable) && \
+ !was_disabled) \
trace_hardirqs_off(); \
} while (0)
#define local_irq_save(flags) \
do { \
raw_local_irq_save(flags); \
- if (!raw_irqs_disabled_flags(flags)) \
+ if (tracepoint_enabled(irq_disable) && \
+ !raw_irqs_disabled_flags(flags)) \
trace_hardirqs_off(); \
} while (0)
#define local_irq_restore(flags) \
do { \
- if (!raw_irqs_disabled_flags(flags)) \
+ if (tracepoint_enabled(irq_enable) && \
+ !raw_irqs_disabled_flags(flags)) \
trace_hardirqs_on(); \
raw_local_irq_restore(flags); \
} while (0)
-#define safe_halt() \
- do { \
- trace_hardirqs_on(); \
- raw_safe_halt(); \
+#define safe_halt() \
+ do { \
+ if (tracepoint_enabled(irq_enable)) \
+ trace_hardirqs_on(); \
+ raw_safe_halt(); \
} while (0)
diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c
index 0c42b15c3800..90ee65db4516 100644
--- a/kernel/trace/trace_preemptirq.c
+++ b/kernel/trace/trace_preemptirq.c
@@ -111,6 +111,9 @@ void trace_hardirqs_off(void)
}
EXPORT_SYMBOL(trace_hardirqs_off);
NOKPROBE_SYMBOL(trace_hardirqs_off);
+
+EXPORT_TRACEPOINT_SYMBOL(irq_disable);
+EXPORT_TRACEPOINT_SYMBOL(irq_enable);
#endif /* CONFIG_TRACE_IRQFLAGS */
#ifdef CONFIG_TRACE_PREEMPT_TOGGLE
--
2.50.0
Hi Wander,
kernel test robot noticed the following build errors:
[auto build test ERROR on trace/for-next]
[also build test ERROR on tip/sched/core linus/master v6.16-rc3 next-20250627]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Wander-Lairson-Costa/trace-preemptirq-reduce-overhead-of-irq_enable-disable-tracepoints/20250626-222438
base: https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace for-next
patch link: https://lore.kernel.org/r/20250626142017.26372-2-wander%40redhat.com
patch subject: [PATCH 1/2] trace/preemptirq: reduce overhead of irq_enable/disable tracepoints
config: parisc-randconfig-001-20250628 (https://download.01.org/0day-ci/archive/20250628/202506280530.WX885a04-lkp@intel.com/config)
compiler: hppa-linux-gcc (GCC) 8.5.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250628/202506280530.WX885a04-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202506280530.WX885a04-lkp@intel.com/
All error/warnings (new ones prefixed by >>):
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/jump_label.h:299:19: error: implicit declaration of function 'atomic_try_cmpxchg'; did you mean 'raw_cpu_try_cmpxchg'? [-Werror=implicit-function-declaration]
} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));
^~~~~~~~~~~~~~~~~~
include/linux/compiler.h:76:40: note: in definition of macro 'likely'
# define likely(x) __builtin_expect(!!(x), 1)
^
In file included from include/linux/static_key.h:1,
from include/linux/tracepoint-defs.h:12,
from include/linux/irqflags.h:20,
from include/asm-generic/cmpxchg-local.h:6,
from arch/parisc/include/asm/cmpxchg.h:87,
from arch/parisc/include/asm/atomic.h:10,
from include/linux/atomic.h:7,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/jump_label.h: In function 'static_key_slow_dec':
include/linux/jump_label.h:307:2: error: implicit declaration of function 'atomic_dec' [-Werror=implicit-function-declaration]
atomic_dec(&key->enabled);
^~~~~~~~~~
include/linux/jump_label.h: In function 'static_key_enable':
include/linux/jump_label.h:326:3: error: implicit declaration of function 'WARN_ON_ONCE'; did you mean 'WRITE_ONCE'? [-Werror=implicit-function-declaration]
WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
^~~~~~~~~~~~
WRITE_ONCE
include/linux/jump_label.h:329:2: error: implicit declaration of function 'atomic_set'; did you mean 'static_assert'? [-Werror=implicit-function-declaration]
atomic_set(&key->enabled, 1);
^~~~~~~~~~
static_assert
In file included from include/linux/atomic.h:80,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/atomic/atomic-arch-fallback.h: At top level:
include/linux/atomic/atomic-arch-fallback.h:455:1: error: static declaration of 'raw_atomic_read' follows non-static declaration
raw_atomic_read(const atomic_t *v)
^~~~~~~~~~~~~~~
In file included from include/linux/static_key.h:1,
from include/linux/tracepoint-defs.h:12,
from include/linux/irqflags.h:20,
from include/asm-generic/cmpxchg-local.h:6,
from arch/parisc/include/asm/cmpxchg.h:87,
from arch/parisc/include/asm/atomic.h:10,
from include/linux/atomic.h:7,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/jump_label.h:262:9: note: previous implicit declaration of 'raw_atomic_read' was here
return raw_atomic_read(&key->enabled);
^~~~~~~~~~~~~~~
In file included from include/linux/atomic.h:82,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/atomic/atomic-instrumented.h:30:1: error: static declaration of 'atomic_read' follows non-static declaration
atomic_read(const atomic_t *v)
^~~~~~~~~~~
In file included from include/linux/static_key.h:1,
from include/linux/tracepoint-defs.h:12,
from include/linux/irqflags.h:20,
from include/asm-generic/cmpxchg-local.h:6,
from arch/parisc/include/asm/cmpxchg.h:87,
from arch/parisc/include/asm/atomic.h:10,
from include/linux/atomic.h:7,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/jump_label.h:295:6: note: previous implicit declaration of 'atomic_read' was here
v = atomic_read(&key->enabled);
^~~~~~~~~~~
In file included from include/linux/atomic.h:82,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
>> include/linux/atomic/atomic-instrumented.h:65:1: warning: conflicting types for 'atomic_set'
atomic_set(atomic_t *v, int i)
^~~~~~~~~~
include/linux/atomic/atomic-instrumented.h:65:1: error: static declaration of 'atomic_set' follows non-static declaration
In file included from include/linux/static_key.h:1,
from include/linux/tracepoint-defs.h:12,
from include/linux/irqflags.h:20,
from include/asm-generic/cmpxchg-local.h:6,
from arch/parisc/include/asm/cmpxchg.h:87,
from arch/parisc/include/asm/atomic.h:10,
from include/linux/atomic.h:7,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/jump_label.h:329:2: note: previous implicit declaration of 'atomic_set' was here
atomic_set(&key->enabled, 1);
^~~~~~~~~~
In file included from include/linux/atomic.h:82,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
>> include/linux/atomic/atomic-instrumented.h:590:1: warning: conflicting types for 'atomic_dec'
atomic_dec(atomic_t *v)
^~~~~~~~~~
include/linux/atomic/atomic-instrumented.h:590:1: error: static declaration of 'atomic_dec' follows non-static declaration
In file included from include/linux/static_key.h:1,
from include/linux/tracepoint-defs.h:12,
from include/linux/irqflags.h:20,
from include/asm-generic/cmpxchg-local.h:6,
from arch/parisc/include/asm/cmpxchg.h:87,
from arch/parisc/include/asm/atomic.h:10,
from include/linux/atomic.h:7,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/jump_label.h:307:2: note: previous implicit declaration of 'atomic_dec' was here
atomic_dec(&key->enabled);
^~~~~~~~~~
In file included from include/linux/atomic.h:82,
from arch/parisc/include/asm/bitops.h:13,
from include/linux/bitops.h:68,
from include/linux/kernel.h:23,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
>> include/linux/atomic/atomic-instrumented.h:1275:1: error: conflicting types for 'atomic_try_cmpxchg'
atomic_try_cmpxchg(atomic_t *v, int *old, int new)
^~~~~~~~~~~~~~~~~~
In file included from include/linux/array_size.h:5,
from include/linux/kernel.h:16,
from arch/parisc/include/asm/bug.h:5,
from include/linux/bug.h:5,
from include/linux/page-flags.h:10,
from kernel/bounds.c:10:
include/linux/jump_label.h:299:19: note: previous implicit declaration of 'atomic_try_cmpxchg' was here
} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));
^~~~~~~~~~~~~~~~~~
include/linux/compiler.h:76:40: note: in definition of macro 'likely'
# define likely(x) __builtin_expect(!!(x), 1)
^
cc1: some warnings being treated as errors
make[3]: *** [scripts/Makefile.build:98: kernel/bounds.s] Error 1 shuffle=1288049444
make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1274: prepare0] Error 2 shuffle=1288049444
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:248: __sub-make] Error 2 shuffle=1288049444
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:248: __sub-make] Error 2 shuffle=1288049444
make: Target 'prepare' not remade because of errors.
vim +/atomic_try_cmpxchg +1275 include/linux/atomic/atomic-instrumented.h
aa525d063851a9 include/asm-generic/atomic-instrumented.h Mark Rutland 2018-09-04 1259
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1260 /**
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1261 * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1262 * @v: pointer to atomic_t
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1263 * @old: pointer to int value to compare with
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1264 * @new: int value to assign
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1265 *
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1266 * If (@v == @old), atomically updates @v to @new with full ordering.
6dfee110c6cc7a include/linux/atomic/atomic-instrumented.h Mark Rutland 2024-02-09 1267 * Otherwise, @v is not modified, @old is updated to the current value of @v,
6dfee110c6cc7a include/linux/atomic/atomic-instrumented.h Mark Rutland 2024-02-09 1268 * and relaxed ordering is provided.
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1269 *
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1270 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1271 *
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1272 * Return: @true if the exchange occured, @false otherwise.
ad8110706f3811 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1273 */
c020395b6634b7 include/asm-generic/atomic-instrumented.h Marco Elver 2019-11-26 1274 static __always_inline bool
aa525d063851a9 include/asm-generic/atomic-instrumented.h Mark Rutland 2018-09-04 @1275 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
aa525d063851a9 include/asm-generic/atomic-instrumented.h Mark Rutland 2018-09-04 1276 {
e87c4f6642f496 include/linux/atomic/atomic-instrumented.h Marco Elver 2021-11-30 1277 kcsan_mb();
3570a1bcf45e9a include/asm-generic/atomic-instrumented.h Marco Elver 2020-07-24 1278 instrument_atomic_read_write(v, sizeof(*v));
3570a1bcf45e9a include/asm-generic/atomic-instrumented.h Marco Elver 2020-07-24 1279 instrument_atomic_read_write(old, sizeof(*old));
c9268ac615f9f6 include/linux/atomic/atomic-instrumented.h Mark Rutland 2023-06-05 1280 return raw_atomic_try_cmpxchg(v, old, new);
aa525d063851a9 include/asm-generic/atomic-instrumented.h Mark Rutland 2018-09-04 1281 }
aa525d063851a9 include/asm-generic/atomic-instrumented.h Mark Rutland 2018-09-04 1282
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
On Thu, 26 Jun 2025 11:20:09 -0300
Wander Lairson Costa <wander@redhat.com> wrote:
> @@ -197,9 +198,13 @@ extern void warn_bogus_irq_restore(void);
> */
> #ifdef CONFIG_TRACE_IRQFLAGS
>
> +DECLARE_TRACEPOINT(irq_enable);
> +DECLARE_TRACEPOINT(irq_disable);
> +
> #define local_irq_enable() \
> do { \
> - trace_hardirqs_on(); \
> + if (tracepoint_enabled(irq_enable)) \
> + trace_hardirqs_on(); \
If you have both this and lockdep enabled, then this has to be called
regardless if tracing is enabled or not. So this should be:
if (IS_ENABLED(CONFIG_PROVE_LOCKING) || \
tracepoint_enabled(irq_enable))
> raw_local_irq_enable(); \
> } while (0)
>
> @@ -207,28 +212,32 @@ extern void warn_bogus_irq_restore(void);
> do { \
> bool was_disabled = raw_irqs_disabled();\
> raw_local_irq_disable(); \
> - if (!was_disabled) \
> + if (tracepoint_enabled(irq_disable) && \
> + !was_disabled) \
And this should be:
if (IS_ENABLED(CONFIG_PROVE_LOCKING) || \
(tracepoint_enabled(irq_disable) && \
!was_disabled))
> trace_hardirqs_off(); \
> } while (0)
>
> #define local_irq_save(flags) \
> do { \
> raw_local_irq_save(flags); \
> - if (!raw_irqs_disabled_flags(flags)) \
> + if (tracepoint_enabled(irq_disable) && \
> + !raw_irqs_disabled_flags(flags)) \
> trace_hardirqs_off(); \
> } while (0)
>
> #define local_irq_restore(flags) \
> do { \
> - if (!raw_irqs_disabled_flags(flags)) \
> + if (tracepoint_enabled(irq_enable) && \
> + !raw_irqs_disabled_flags(flags)) \
> trace_hardirqs_on(); \
Same with these.
-- Steve
> raw_local_irq_restore(flags); \
> } while (0)
>
> -#define safe_halt() \
> - do { \
> - trace_hardirqs_on(); \
> - raw_safe_halt(); \
> +#define safe_halt() \
> + do { \
> + if (tracepoint_enabled(irq_enable)) \
> + trace_hardirqs_on(); \
> + raw_safe_halt(); \
> } while (0)
>
>
> diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c
> index 0c42b15c3800..90ee65db4516 100644
> --- a/kernel/trace/trace_preemptirq.c
> +++ b/kernel/trace/trace_preemptirq.c
> @@ -111,6 +111,9 @@ void trace_hardirqs_off(void)
> }
> EXPORT_SYMBOL(trace_hardirqs_off);
> NOKPROBE_SYMBOL(trace_hardirqs_off);
> +
> +EXPORT_TRACEPOINT_SYMBOL(irq_disable);
> +EXPORT_TRACEPOINT_SYMBOL(irq_enable);
> #endif /* CONFIG_TRACE_IRQFLAGS */
>
> #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
© 2016 - 2025 Red Hat, Inc.