.../admin-guide/kernel-parameters.txt | 10 +++++++++ include/linux/printk.h | 10 +++++++++ include/linux/sys_info.h | 3 +++ kernel/panic.c | 21 +++++++++++++++++-- kernel/printk/internal.h | 4 ++++ kernel/printk/printk.c | 9 ++++++++ kernel/printk/printk_ringbuffer.c | 18 ++++++++++++++++ lib/sys_info.c | 20 +++++++++--------- 8 files changed, 83 insertions(+), 12 deletions(-)
When investigating kernel panics, capturing post-mortem diagnostic
telemetry (e.g. memory zone metrics, lock states, active timers, and
blocked tasks) is vital for root-cause analysis.
While crash_kexec_post_notifiers allows executing panic notifiers and
sys_info() before jumping to the kdump kernel, it is frequently avoided
in production environments due to the risk of watchdog timeouts induced
by synchronous hardware console emission.
To resolve this dilemma, introduce the crash_kexec_in_memory_sys_info
boot parameter. When enabled, it captures diagnostic telemetry directly
into the printk ring buffer entirely in RAM before jumping to
__crash_kexec(), completing in milliseconds rather than tens of seconds.
To make this safe, fast, and reliable without risking buffer overflow:
1. Scoped console flush suppression
Provide printk_suppress_console_flush(bool) to clear the console
flush mask in printk_get_console_flush_type(). Messages written
via vprintk_store() remain in the printk ring buffer in memory
and avoid synchronous hardware console emission and waking
kthreads.
2. Scoped ring buffer tail freezing
Introduce printk_freeze_tail(bool) in printk_ringbuffer. When
active, desc_push_tail() and data_push_tail() refuse to advance
the tail. If diagnostic logging exhausts available ring buffer
headroom, new records are safely dropped, guaranteeing that the
initial panic Oops, faulting registers, and primary stack trace
are never overwritten.
3. Execution sequence reordering
Reorder __sys_info() so compact, high-signal subsystems (e.g.
memory) are collected first, leaving high-volume dumps (all CPU
backtraces, full task lists, and ftrace) for last.
4. Sensible defaults and fail-safe fallback
Default to SYS_INFO_IN_MEMORY_DEFAULT if panic_sys_info is not
explicitly configured. If __crash_kexec() returns or fails to
execute, tail freezing, and console flush suppression are
unmasked immediately so emergency output can flush to physical
consoles.
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
.../admin-guide/kernel-parameters.txt | 10 +++++++++
include/linux/printk.h | 10 +++++++++
include/linux/sys_info.h | 3 +++
kernel/panic.c | 21 +++++++++++++++++--
kernel/printk/internal.h | 4 ++++
kernel/printk/printk.c | 9 ++++++++
kernel/printk/printk_ringbuffer.c | 18 ++++++++++++++++
lib/sys_info.c | 20 +++++++++---------
8 files changed, 83 insertions(+), 12 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index eb608e5139a6..2b7ee0892151 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1037,6 +1037,16 @@ Kernel parameters
Default is enabled if CONFIG_HOTPLUG_PARALLEL=y. Otherwise
the parameter has no effect.
+ crash_kexec_in_memory_sys_info
+ [KNL] Collect diagnostic system telemetry (memory,
+ locks, timers, blocked tasks) into the
+ printk ring buffer entirely in RAM before jumping to
+ the kdump kernel. Unlike crash_kexec_post_notifiers,
+ this option bypasses slow hardware console emission and
+ avoids running external panic notifiers. The printk ring
+ buffer tail is also frozen during collection to prevent
+ telemetry output from overwriting the initial panic Oops.
+
crash_kexec_post_notifiers
Only jump to kdump kernel after running the panic
notifiers and dumping kmsg. This option increases
diff --git a/include/linux/printk.h b/include/linux/printk.h
index f594c1266bfd..b046e51803a7 100644
--- a/include/linux/printk.h
+++ b/include/linux/printk.h
@@ -205,6 +205,8 @@ extern asmlinkage void dump_stack(void) __cold;
void printk_trigger_flush(void);
void console_try_replay_all(void);
void printk_legacy_allow_panic_sync(void);
+void printk_suppress_console_flush(bool suppress);
+void printk_freeze_tail(bool freeze);
extern bool nbcon_device_try_acquire(struct console *con);
extern void nbcon_device_release(struct console *con);
void nbcon_atomic_flush_unsafe(void);
@@ -309,6 +311,14 @@ static inline void printk_legacy_allow_panic_sync(void)
{
}
+static inline void printk_suppress_console_flush(bool suppress)
+{
+}
+
+static inline void printk_freeze_tail(bool freeze)
+{
+}
+
static inline bool nbcon_device_try_acquire(struct console *con)
{
return false;
diff --git a/include/linux/sys_info.h b/include/linux/sys_info.h
index a5bc3ea3d44b..1f58bdaed945 100644
--- a/include/linux/sys_info.h
+++ b/include/linux/sys_info.h
@@ -17,6 +17,9 @@
#define SYS_INFO_ALL_BT 0x00000040
#define SYS_INFO_BLOCKED_TASKS 0x00000080
+#define SYS_INFO_IN_MEMORY_DEFAULT (SYS_INFO_MEM | SYS_INFO_LOCKS | \
+ SYS_INFO_TIMERS | SYS_INFO_BLOCKED_TASKS)
+
void sys_info(unsigned long si_mask);
unsigned long sys_info_parse_param(char *str);
diff --git a/kernel/panic.c b/kernel/panic.c
index 213725b612aa..a7e2101dd968 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -62,6 +62,7 @@ static int pause_on_oops;
static int pause_on_oops_flag;
static DEFINE_SPINLOCK(pause_on_oops_lock);
bool crash_kexec_post_notifiers;
+static bool crash_kexec_in_memory_sys_info;
int panic_on_warn __read_mostly;
unsigned long panic_on_taint;
bool panic_on_taint_nousertaint = false;
@@ -580,6 +581,7 @@ void vpanic(const char *fmt, va_list args)
long i, i_next = 0, len;
int state = 0;
bool _crash_kexec_post_notifiers = crash_kexec_post_notifiers;
+ bool _crash_kexec_in_memory_sys_info = crash_kexec_in_memory_sys_info;
if (panic_on_warn) {
/*
@@ -667,8 +669,22 @@ void vpanic(const char *fmt, va_list args)
*
* Bypass the panic_cpu check and call __crash_kexec directly.
*/
- if (!_crash_kexec_post_notifiers)
- __crash_kexec(NULL);
+ if (!_crash_kexec_post_notifiers) {
+ if (_crash_kexec_in_memory_sys_info) {
+ unsigned long si_mask = panic_print ? : SYS_INFO_IN_MEMORY_DEFAULT;
+
+ /* Populate log_buf in RAM without stalling on slow UARTs */
+ printk_suppress_console_flush(true);
+ printk_freeze_tail(true);
+ sys_info(si_mask);
+ kmsg_dump_desc(KMSG_DUMP_PANIC, buf);
+ __crash_kexec(NULL);
+ printk_freeze_tail(false);
+ printk_suppress_console_flush(false);
+ } else {
+ __crash_kexec(NULL);
+ }
+ }
panic_other_cpus_shutdown(_crash_kexec_post_notifiers);
@@ -1217,6 +1233,7 @@ core_param(panic, panic_timeout, int, 0644);
core_param(pause_on_oops, pause_on_oops, int, 0644);
core_param(panic_on_warn, panic_on_warn, int, 0644);
core_param(crash_kexec_post_notifiers, crash_kexec_post_notifiers, bool, 0644);
+core_param(crash_kexec_in_memory_sys_info, crash_kexec_in_memory_sys_info, bool, 0644);
core_param(panic_console_replay, panic_console_replay, bool, 0644);
static int panic_print_set(const char *val, const struct kernel_param *kp)
diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h
index 85fbf1801cbe..72f0a5d154ec 100644
--- a/kernel/printk/internal.h
+++ b/kernel/printk/internal.h
@@ -186,6 +186,7 @@ struct console_flush_type {
};
extern bool console_irqwork_blocked;
+extern bool console_flush_suppressed;
/*
* Identify which console flushing methods should be used in the context of
@@ -195,6 +196,9 @@ static inline void printk_get_console_flush_type(struct console_flush_type *ft)
{
memset(ft, 0, sizeof(*ft));
+ if (unlikely(READ_ONCE(console_flush_suppressed)))
+ return;
+
switch (nbcon_get_default_prio()) {
case NBCON_PRIO_NORMAL:
if (have_nbcon_console && !have_boot_console) {
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 3fcdf4b4e2e5..716eb692c5fa 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -467,6 +467,15 @@ bool legacy_allow_panic_sync;
/* Avoid using irq_work when suspending. */
bool console_irqwork_blocked;
+bool console_flush_suppressed;
+EXPORT_SYMBOL_GPL(console_flush_suppressed);
+
+void printk_suppress_console_flush(bool suppress)
+{
+ WRITE_ONCE(console_flush_suppressed, suppress);
+}
+EXPORT_SYMBOL_GPL(printk_suppress_console_flush);
+
#ifdef CONFIG_PRINTK
DECLARE_WAIT_QUEUE_HEAD(log_wait);
static DECLARE_WAIT_QUEUE_HEAD(legacy_wait);
diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c
index 85c0c854b3ce..f538c942a349 100644
--- a/kernel/printk/printk_ringbuffer.c
+++ b/kernel/printk/printk_ringbuffer.c
@@ -377,6 +377,14 @@ static struct prb_data_block *to_block(struct prb_data_ring *data_ring,
return (void *)&data_ring->data[DATA_INDEX(data_ring, begin_lpos)];
}
+static bool printk_tail_frozen;
+
+void printk_freeze_tail(bool freeze)
+{
+ WRITE_ONCE(printk_tail_frozen, freeze);
+}
+EXPORT_SYMBOL_GPL(printk_freeze_tail);
+
/*
* Increase the data size to account for data block meta data plus any
* padding so that the adjacent data block is aligned on the ID size.
@@ -678,6 +686,10 @@ static bool data_push_tail(struct printk_ringbuffer *rb, unsigned long lpos)
*/
tail_lpos = atomic_long_read(&data_ring->tail_lpos); /* LMM(data_push_tail:A) */
+ if (unlikely(READ_ONCE(printk_tail_frozen)) &&
+ need_more_space(data_ring, tail_lpos, lpos))
+ return false;
+
/*
* Loop until the tail lpos is at or beyond @lpos. This condition
* may already be satisfied, resulting in no full memory barrier
@@ -789,6 +801,9 @@ static bool desc_push_tail(struct printk_ringbuffer *rb,
enum desc_state d_state;
struct prb_desc desc;
+ if (unlikely(READ_ONCE(printk_tail_frozen)))
+ return false;
+
d_state = desc_read(desc_ring, tail_id, &desc, NULL, NULL);
switch (d_state) {
@@ -935,6 +950,9 @@ static bool desc_reserve(struct printk_ringbuffer *rb, unsigned long *id_out)
* Make space for the new descriptor by
* advancing the tail.
*/
+ if (unlikely(READ_ONCE(printk_tail_frozen)))
+ return false;
+
if (!desc_push_tail(rb, id_prev_wrap))
return false;
}
diff --git a/lib/sys_info.c b/lib/sys_info.c
index f32a06ec9ed4..06a2e7b0c87b 100644
--- a/lib/sys_info.c
+++ b/lib/sys_info.c
@@ -138,26 +138,26 @@ subsys_initcall(sys_info_sysctl_init);
static void __sys_info(unsigned long si_mask)
{
- if (si_mask & SYS_INFO_TASKS)
- show_state();
-
if (si_mask & SYS_INFO_MEM)
show_mem();
- if (si_mask & SYS_INFO_TIMERS)
- sysrq_timer_list_show();
-
if (si_mask & SYS_INFO_LOCKS)
debug_show_all_locks();
- if (si_mask & SYS_INFO_FTRACE)
- ftrace_dump(DUMP_ALL);
+ if (si_mask & SYS_INFO_TIMERS)
+ sysrq_timer_list_show();
+
+ if (si_mask & SYS_INFO_BLOCKED_TASKS)
+ show_state_filter(TASK_UNINTERRUPTIBLE);
if (si_mask & SYS_INFO_ALL_BT)
trigger_all_cpu_backtrace();
- if (si_mask & SYS_INFO_BLOCKED_TASKS)
- show_state_filter(TASK_UNINTERRUPTIBLE);
+ if (si_mask & SYS_INFO_TASKS)
+ show_state();
+
+ if (si_mask & SYS_INFO_FTRACE)
+ ftrace_dump(DUMP_ALL);
}
void sys_info(unsigned long si_mask)
--
2.55.0Adding Guilherme and John into Cc.
On Wed 2026-09-16 16:15:46, Aaron Tomlin wrote:
> When investigating kernel panics, capturing post-mortem diagnostic
> telemetry (e.g. memory zone metrics, lock states, active timers, and
> blocked tasks) is vital for root-cause analysis.
>
> While crash_kexec_post_notifiers allows executing panic notifiers and
> sys_info() before jumping to the kdump kernel, it is frequently avoided
> in production environments due to the risk of watchdog timeouts induced
> by synchronous hardware console emission.
There seems to be various motivations to set/clear
crash_kexec_post_notifiers.
Guilherme wanted to add some filtering because some notifiers
were failing, see
see https://lore.kernel.org/all/20220108153451.195121-1-gpiccoli@igalia.com/
I believe that they are called after kdump by default because
the information provided by them is included in the dump.
This idea is supported by the commit f06e5153f4ae2e2f3
("kernel/panic.c: add "crash_kexec_post_notifiers" option
for kdump after panic_notifers").
On the other hand, crash_kexec_post_notifiers is explicitely when
the kernel is running on some hypervisors because the hypervisors
need to get notified about the panic() before crash dump.
All I want to say is that the situation around
crash_kexec_post_notifiers is much more complicated. And I hear
about the watchdog timeouts in this context for the first time.
> To resolve this dilemma, introduce the crash_kexec_in_memory_sys_info
> boot parameter. When enabled, it captures diagnostic telemetry directly
> into the printk ring buffer entirely in RAM before jumping to
> __crash_kexec(), completing in milliseconds rather than tens of seconds.
>
> To make this safe, fast, and reliable without risking buffer overflow:
> 1. Scoped console flush suppression
>
> Provide printk_suppress_console_flush(bool) to clear the console
> flush mask in printk_get_console_flush_type(). Messages written
> via vprintk_store() remain in the printk ring buffer in memory
> and avoid synchronous hardware console emission and waking
> kthreads.
This might help when the claim about watchdog reports is true.
I am not sure about it. Anyway, there are other ways how to
prevent watchdogs stepping in (touching them, disabling them, ...)
The console output is important when the crashdump fails.
> 2. Scoped ring buffer tail freezing
>
> Introduce printk_freeze_tail(bool) in printk_ringbuffer. When
> active, desc_push_tail() and data_push_tail() refuse to advance
> the tail. If diagnostic logging exhausts available ring buffer
> headroom, new records are safely dropped, guaranteeing that the
> initial panic Oops, faulting registers, and primary stack trace
> are never overwritten.
This is another questionable feature. The ring buffer would need to be
super big to hold all messages since the boot. IMHO, servers are
normally running hundreds of days and the log buffer gets rotated,
like the user space logs, ...
> 3. Execution sequence reordering
>
> Reorder __sys_info() so compact, high-signal subsystems (e.g.
> memory) are collected first, leaving high-volume dumps (all CPU
> backtraces, full task lists, and ftrace) for last.
This might make sense. I am just afraid that it might be a personal
opinion and we might end up with an endless shuffling here.
My opinion:
IMHO, it does not make much sense to dump sys_info() before kdump
and block consoles. The information is lost when kdump fails.
The information can be extracted from the crashdump when
kdump succeeds.
Best Regards,
Petr
On Tue, Sep 22, 2026 at 05:29:45PM +0200, Petr Mladek wrote:
> Adding Guilherme and John into Cc.
>
> On Wed 2026-09-16 16:15:46, Aaron Tomlin wrote:
> > When investigating kernel panics, capturing post-mortem diagnostic
> > telemetry (e.g. memory zone metrics, lock states, active timers, and
> > blocked tasks) is vital for root-cause analysis.
> >
> > While crash_kexec_post_notifiers allows executing panic notifiers and
> > sys_info() before jumping to the kdump kernel, it is frequently avoided
> > in production environments due to the risk of watchdog timeouts induced
> > by synchronous hardware console emission.
>
> There seems to be various motivations to set/clear
> crash_kexec_post_notifiers.
>
> Guilherme wanted to add some filtering because some notifiers
> were failing, see
> see https://lore.kernel.org/all/20220108153451.195121-1-gpiccoli@igalia.com/
>
> I believe that they are called after kdump by default because
> the information provided by them is included in the dump.
> This idea is supported by the commit f06e5153f4ae2e2f3
> ("kernel/panic.c: add "crash_kexec_post_notifiers" option
> for kdump after panic_notifers").
>
> On the other hand, crash_kexec_post_notifiers is explicitely when
> the kernel is running on some hypervisors because the hypervisors
> need to get notified about the panic() before crash dump.
>
> All I want to say is that the situation around
> crash_kexec_post_notifiers is much more complicated. And I hear
> about the watchdog timeouts in this context for the first time.
Hi Petr,
Thank you for your feedback.
Interesting. Guilherme's work on classifying and filtering panic notifiers
sought to address a well-known vulnerability: namely, that third-party or
device driver callbacks registered on panic_notifier_list can be fragile,
corrupt state, or hang the crashed kernel before kdump has had an
opportunity to boot.
However, my proposal here is quite distinct from Guilherme's effort, though
it shares the goal of making pre-kdump execution dependable.
1. Bypassing panic notifier entirely
I deliberately avoid invoking panic_notifier_list. The instability
of arbitrary notifier callbacks that Guilherme highlighted is
precisely why some operators avoid crash_kexec_post_notifiers in
production. Our intent is strictly to collect well-defined kernel
state via sys_info()
2. Purely in-memory collection
This patch addresses both dilemmas simultaneously:
- It eliminates the risk of notifier instability by not
invoking panic_notifier_list.
- It eliminates console stalls by suppressing console emission
and freezing the printk tail, recording the telemetry purely
into the printk ring buffer in RAM (within milliseconds)
before jumping to __crash_kexec().
Consequently, the capture kernel receives an enriched log_buf (extractable
via vmcore-dmesg.txt or the vmcore) with virtually zero risk of hardware
watchdog trips or notifier lockups.
> > To resolve this dilemma, introduce the crash_kexec_in_memory_sys_info
> > boot parameter. When enabled, it captures diagnostic telemetry directly
> > into the printk ring buffer entirely in RAM before jumping to
> > __crash_kexec(), completing in milliseconds rather than tens of seconds.
> >
> > To make this safe, fast, and reliable without risking buffer overflow:
> > 1. Scoped console flush suppression
> >
> > Provide printk_suppress_console_flush(bool) to clear the console
> > flush mask in printk_get_console_flush_type(). Messages written
> > via vprintk_store() remain in the printk ring buffer in memory
> > and avoid synchronous hardware console emission and waking
> > kthreads.
>
> This might help when the claim about watchdog reports is true.
> I am not sure about it. Anyway, there are other ways how to
> prevent watchdogs stepping in (touching them, disabling them, ...)
Regarding watchdog intervention, the concern does not stem from internal
kernel watchdogs (e.g. softlockup watchdog), but rather from autonomous
vendor-specific hardware/platform watchdogs (e.g. hpwdt).
A situation occurs with IPMI watchdog
(i.e. drivers/char/ipmi/ipmi_watchdog.c) configured with pre-timeouts,
where the BMC asserts an NMI and independently enforces a hard reset after
a brief remaining window.
In these configurations:
- "Touching" the watchdog is ineffective i.e. touch_nmi_watchdog() has
no awareness of external hardware or BMC timers.
- Some production environments mandate nowayout=1
(drivers/char/ipmi/ipmi_watchdog.c), which deliberately forbids
software from stopping the timer.
On a slow serial console or IPMI Serial-over-LAN (e.g. 115200 baud),
flushing comprehensive sys_info() output (e.g. tasks or stacktraces) can
exceed 10 seconds, allowing the hardware timer to expire and reset the
system before kdump can initialise.
Regarding your remark that console output is important when crashdump
fails, I completely agree. That scenario is explicitly handled:
sys_info(si_mask);
kmsg_dump_desc(KMSG_DUMP_PANIC, buf);
__crash_kexec(NULL);
printk_freeze_tail(false);
printk_suppress_console_flush(false);
If __crash_kexec() fails or is not armed, suppression is immediately
unmasked before proceeding down the remainder of panic(). As a result,
console_flush_on_panic() flushes the complete buffer, including both the
panic Oops and the newly gathered sys_info telemetry, out to the physical
console as normal.
To clarify, an independent, external hardware controller will forcibly
reboot the machine if the kernel spends too long flushing diagnostic logs
to slow physical serial lines before jumping to __crash_kexec().
>
> The console output is important when the crashdump fails.
>
> > 2. Scoped ring buffer tail freezing
> >
> > Introduce printk_freeze_tail(bool) in printk_ringbuffer. When
> > active, desc_push_tail() and data_push_tail() refuse to advance
> > the tail. If diagnostic logging exhausts available ring buffer
> > headroom, new records are safely dropped, guaranteeing that the
> > initial panic Oops, faulting registers, and primary stack trace
> > are never overwritten.
>
> This is another questionable feature. The ring buffer would need to be
> super big to hold all messages since the boot. IMHO, servers are
> normally running hundreds of days and the log buffer gets rotated,
> like the user space logs, ...
Apologies for the misunderstanding regarding the lifecycle of the freeze.
Please note, newly proposed printk_freeze_tail() is not active from boot,
nor does it hinder normal buffer rotation while the system is running. It
is activated strictly for the few milliseconds during which sys_info()
executes:
printk_freeze_tail(true);
sys_info(si_mask);
printk_freeze_tail(false);
Prior to this call, the panic Oops, registers, and faulting stack trace
have already been logged. The sole purpose of freezing the tail during
sys_info() is to prevent newly generated telemetry from wrapping the buffer
and overwriting that initial, critical panic Oops if the remaining headroom
is exhausted.
> > 3. Execution sequence reordering
> >
> > Reorder __sys_info() so compact, high-signal subsystems (e.g.
> > memory) are collected first, leaving high-volume dumps (all CPU
> > backtraces, full task lists, and ftrace) for last.
>
> This might make sense. I am just afraid that it might be a personal
> opinion and we might end up with an endless shuffling here.
Fair point. The intention was merely to prioritise compact, high-density
metrics before unbounded dumps (e.g. all-task backtraces). However, if this
introduces unwanted churn, I am entirely content to drop the reordering and
retain the existing sequence.
> My opinion:
>
> IMHO, it does not make much sense to dump sys_info() before kdump
> and block consoles. The information is lost when kdump fails.
> The information can be extracted from the crashdump when
> kdump succeeds.
The information is not lost if kdump fails, as suppression is immediately
lifted. In an ideal world, the full vmcore (i.e. makedumpfile -d 31) is
always saved intact. In production, however, dump targets may suffer from
storage exhaustion (e.g. accumulated prior vmcores). When this occurs,
writing the multi-gigabyte /proc/vmcore is truncated mid-stream, leaving
the memory dump corrupt or too incomplete to extract dmesg e.g. via crash
or drgn.
Crucially, kdump utilities (e.g. makedumpfile --dump-dmesg) extract
vmcore-dmesg.txt first. Because it requires only a few megabytes, it
reliably succeeds even when saving the full vmcore fails due to ENOSPC.
Capturing sys_info() in-memory beforehand guarantees that vital state
remains preserved in vmcore-dmesg.txt as the sole surviving diagnostic
artefact.
Kind regards,
--
Aaron Tomlin
On 16 September 2026 21:15:46 BST, Aaron Tomlin <atomlin@atomlin.com>
wrote:
>When investigating kernel panics, capturing post-mortem diagnostic
>telemetry (e.g. memory zone metrics, lock states, active timers, and
>blocked tasks) is vital for root-cause analysis.
>
>While crash_kexec_post_notifiers allows executing panic notifiers and
>sys_info() before jumping to the kdump kernel, it is frequently avoided
>in production environments due to the risk of watchdog timeouts induced
>by synchronous hardware console emission.
>
>To resolve this dilemma, introduce the crash_kexec_in_memory_sys_info
>boot parameter. When enabled, it captures diagnostic telemetry directly
>into the printk ring buffer entirely in RAM before jumping to
>__crash_kexec(), completing in milliseconds rather than tens of seconds.
>
>To make this safe, fast, and reliable without risking buffer overflow:
> 1. Scoped console flush suppression
>
> Provide printk_suppress_console_flush(bool) to clear the console
> flush mask in printk_get_console_flush_type(). Messages written
> via vprintk_store() remain in the printk ring buffer in memory
> and avoid synchronous hardware console emission and waking
> kthreads.
>
> 2. Scoped ring buffer tail freezing
>
> Introduce printk_freeze_tail(bool) in printk_ringbuffer. When
> active, desc_push_tail() and data_push_tail() refuse to advance
> the tail. If diagnostic logging exhausts available ring buffer
> headroom, new records are safely dropped, guaranteeing that the
> initial panic Oops, faulting registers, and primary stack trace
> are never overwritten.
>
> 3. Execution sequence reordering
>
> Reorder __sys_info() so compact, high-signal subsystems (e.g.
> memory) are collected first, leaving high-volume dumps (all CPU
> backtraces, full task lists, and ftrace) for last.
>
> 4. Sensible defaults and fail-safe fallback
>
> Default to SYS_INFO_IN_MEMORY_DEFAULT if panic_sys_info is not
> explicitly configured. If __crash_kexec() returns or fails to
> execute, tail freezing, and console flush suppression are
> unmasked immediately so emergency output can flush to physical
> consoles.
>
This looks fun, might review later.
>Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
>---
> .../admin-guide/kernel-parameters.txt | 10 +++++++++
> include/linux/printk.h | 10 +++++++++
> include/linux/sys_info.h | 3 +++
> kernel/panic.c | 21 +++++++++++++++++--
> kernel/printk/internal.h | 4 ++++
> kernel/printk/printk.c | 9 ++++++++
> kernel/printk/printk_ringbuffer.c | 18 ++++++++++++++++
> lib/sys_info.c | 20 +++++++++---------
> 8 files changed, 83 insertions(+), 12 deletions(-)
>
>diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
>index eb608e5139a6..2b7ee0892151 100644
>--- a/Documentation/admin-guide/kernel-parameters.txt
>+++ b/Documentation/admin-guide/kernel-parameters.txt
>@@ -1037,6 +1037,16 @@ Kernel parameters
> Default is enabled if CONFIG_HOTPLUG_PARALLEL=y. Otherwise
> the parameter has no effect.
>
>+ crash_kexec_in_memory_sys_info
>+ [KNL] Collect diagnostic system telemetry (memory,
>+ locks, timers, blocked tasks) into the
>+ printk ring buffer entirely in RAM before jumping to
>+ the kdump kernel. Unlike crash_kexec_post_notifiers,
>+ this option bypasses slow hardware console emission and
>+ avoids running external panic notifiers. The printk ring
>+ buffer tail is also frozen during collection to prevent
>+ telemetry output from overwriting the initial panic Oops.
>+
> crash_kexec_post_notifiers
> Only jump to kdump kernel after running the panic
> notifiers and dumping kmsg. This option increases
>diff --git a/include/linux/printk.h b/include/linux/printk.h
>index f594c1266bfd..b046e51803a7 100644
>--- a/include/linux/printk.h
>+++ b/include/linux/printk.h
>@@ -205,6 +205,8 @@ extern asmlinkage void dump_stack(void) __cold;
> void printk_trigger_flush(void);
> void console_try_replay_all(void);
> void printk_legacy_allow_panic_sync(void);
>+void printk_suppress_console_flush(bool suppress);
>+void printk_freeze_tail(bool freeze);
> extern bool nbcon_device_try_acquire(struct console *con);
> extern void nbcon_device_release(struct console *con);
> void nbcon_atomic_flush_unsafe(void);
>@@ -309,6 +311,14 @@ static inline void printk_legacy_allow_panic_sync(void)
> {
> }
>
>+static inline void printk_suppress_console_flush(bool suppress)
>+{
>+}
>+
>+static inline void printk_freeze_tail(bool freeze)
>+{
>+}
>+
> static inline bool nbcon_device_try_acquire(struct console *con)
> {
> return false;
>diff --git a/include/linux/sys_info.h b/include/linux/sys_info.h
>index a5bc3ea3d44b..1f58bdaed945 100644
>--- a/include/linux/sys_info.h
>+++ b/include/linux/sys_info.h
>@@ -17,6 +17,9 @@
> #define SYS_INFO_ALL_BT 0x00000040
> #define SYS_INFO_BLOCKED_TASKS 0x00000080
>
>+#define SYS_INFO_IN_MEMORY_DEFAULT (SYS_INFO_MEM | SYS_INFO_LOCKS | \
>+ SYS_INFO_TIMERS | SYS_INFO_BLOCKED_TASKS)
>+
> void sys_info(unsigned long si_mask);
> unsigned long sys_info_parse_param(char *str);
>
>diff --git a/kernel/panic.c b/kernel/panic.c
>index 213725b612aa..a7e2101dd968 100644
>--- a/kernel/panic.c
>+++ b/kernel/panic.c
>@@ -62,6 +62,7 @@ static int pause_on_oops;
> static int pause_on_oops_flag;
> static DEFINE_SPINLOCK(pause_on_oops_lock);
> bool crash_kexec_post_notifiers;
>+static bool crash_kexec_in_memory_sys_info;
> int panic_on_warn __read_mostly;
> unsigned long panic_on_taint;
> bool panic_on_taint_nousertaint = false;
>@@ -580,6 +581,7 @@ void vpanic(const char *fmt, va_list args)
> long i, i_next = 0, len;
> int state = 0;
> bool _crash_kexec_post_notifiers = crash_kexec_post_notifiers;
>+ bool _crash_kexec_in_memory_sys_info = crash_kexec_in_memory_sys_info;
>
> if (panic_on_warn) {
> /*
>@@ -667,8 +669,22 @@ void vpanic(const char *fmt, va_list args)
> *
> * Bypass the panic_cpu check and call __crash_kexec directly.
> */
>- if (!_crash_kexec_post_notifiers)
>- __crash_kexec(NULL);
>+ if (!_crash_kexec_post_notifiers) {
>+ if (_crash_kexec_in_memory_sys_info) {
>+ unsigned long si_mask = panic_print ? : SYS_INFO_IN_MEMORY_DEFAULT;
>+
>+ /* Populate log_buf in RAM without stalling on slow UARTs */
>+ printk_suppress_console_flush(true);
>+ printk_freeze_tail(true);
>+ sys_info(si_mask);
>+ kmsg_dump_desc(KMSG_DUMP_PANIC, buf);
>+ __crash_kexec(NULL);
>+ printk_freeze_tail(false);
>+ printk_suppress_console_flush(false);
>+ } else {
>+ __crash_kexec(NULL);
>+ }
>+ }
>
> panic_other_cpus_shutdown(_crash_kexec_post_notifiers);
>
>@@ -1217,6 +1233,7 @@ core_param(panic, panic_timeout, int, 0644);
> core_param(pause_on_oops, pause_on_oops, int, 0644);
> core_param(panic_on_warn, panic_on_warn, int, 0644);
> core_param(crash_kexec_post_notifiers, crash_kexec_post_notifiers, bool,
> 0644);
>+core_param(crash_kexec_in_memory_sys_info, crash_kexec_in_memory_sys_info, bool, 0644);
> core_param(panic_console_replay, panic_console_replay, bool, 0644);
>
> static int panic_print_set(const char *val, const struct kernel_param
> *kp)
>diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h
>index 85fbf1801cbe..72f0a5d154ec 100644
>--- a/kernel/printk/internal.h
>+++ b/kernel/printk/internal.h
>@@ -186,6 +186,7 @@ struct console_flush_type {
> };
>
> extern bool console_irqwork_blocked;
>+extern bool console_flush_suppressed;
>
> /*
> * Identify which console flushing methods should be used in the context of
>@@ -195,6 +196,9 @@ static inline void printk_get_console_flush_type(struct console_flush_type *ft)
> {
> memset(ft, 0, sizeof(*ft));
>
>+ if (unlikely(READ_ONCE(console_flush_suppressed)))
>+ return;
>+
> switch (nbcon_get_default_prio()) {
> case NBCON_PRIO_NORMAL:
> if (have_nbcon_console && !have_boot_console) {
>diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
>index 3fcdf4b4e2e5..716eb692c5fa 100644
>--- a/kernel/printk/printk.c
>+++ b/kernel/printk/printk.c
>@@ -467,6 +467,15 @@ bool legacy_allow_panic_sync;
> /* Avoid using irq_work when suspending. */
> bool console_irqwork_blocked;
>
>+bool console_flush_suppressed;
>+EXPORT_SYMBOL_GPL(console_flush_suppressed);
>+
>+void printk_suppress_console_flush(bool suppress)
>+{
>+ WRITE_ONCE(console_flush_suppressed, suppress);
>+}
>+EXPORT_SYMBOL_GPL(printk_suppress_console_flush);
>+
> #ifdef CONFIG_PRINTK
> DECLARE_WAIT_QUEUE_HEAD(log_wait);
> static DECLARE_WAIT_QUEUE_HEAD(legacy_wait);
>diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c
>index 85c0c854b3ce..f538c942a349 100644
>--- a/kernel/printk/printk_ringbuffer.c
>+++ b/kernel/printk/printk_ringbuffer.c
>@@ -377,6 +377,14 @@ static struct prb_data_block *to_block(struct prb_data_ring *data_ring,
> return (void *)&data_ring->data[DATA_INDEX(data_ring, begin_lpos)];
> }
>
>+static bool printk_tail_frozen;
>+
>+void printk_freeze_tail(bool freeze)
>+{
>+ WRITE_ONCE(printk_tail_frozen, freeze);
>+}
>+EXPORT_SYMBOL_GPL(printk_freeze_tail);
>+
> /*
> * Increase the data size to account for data block meta data plus any
> * padding so that the adjacent data block is aligned on the ID size.
>@@ -678,6 +686,10 @@ static bool data_push_tail(struct printk_ringbuffer *rb, unsigned long lpos)
> */
> tail_lpos = atomic_long_read(&data_ring->tail_lpos); /* LMM(data_push_tail:A) */
>
>+ if (unlikely(READ_ONCE(printk_tail_frozen)) &&
>+ need_more_space(data_ring, tail_lpos, lpos))
>+ return false;
>+
> /*
> * Loop until the tail lpos is at or beyond @lpos. This condition
> * may already be satisfied, resulting in no full memory barrier
>@@ -789,6 +801,9 @@ static bool desc_push_tail(struct printk_ringbuffer *rb,
> enum desc_state d_state;
> struct prb_desc desc;
>
>+ if (unlikely(READ_ONCE(printk_tail_frozen)))
>+ return false;
>+
> d_state = desc_read(desc_ring, tail_id, &desc, NULL, NULL);
>
> switch (d_state) {
>@@ -935,6 +950,9 @@ static bool desc_reserve(struct printk_ringbuffer *rb, unsigned long *id_out)
> * Make space for the new descriptor by
> * advancing the tail.
> */
>+ if (unlikely(READ_ONCE(printk_tail_frozen)))
>+ return false;
>+
> if (!desc_push_tail(rb, id_prev_wrap))
> return false;
> }
>diff --git a/lib/sys_info.c b/lib/sys_info.c
>index f32a06ec9ed4..06a2e7b0c87b 100644
>--- a/lib/sys_info.c
>+++ b/lib/sys_info.c
>@@ -138,26 +138,26 @@ subsys_initcall(sys_info_sysctl_init);
>
> static void __sys_info(unsigned long si_mask)
> {
>- if (si_mask & SYS_INFO_TASKS)
>- show_state();
>-
> if (si_mask & SYS_INFO_MEM)
> show_mem();
>
>- if (si_mask & SYS_INFO_TIMERS)
>- sysrq_timer_list_show();
>-
> if (si_mask & SYS_INFO_LOCKS)
> debug_show_all_locks();
>
>- if (si_mask & SYS_INFO_FTRACE)
>- ftrace_dump(DUMP_ALL);
>+ if (si_mask & SYS_INFO_TIMERS)
>+ sysrq_timer_list_show();
>+
>+ if (si_mask & SYS_INFO_BLOCKED_TASKS)
>+ show_state_filter(TASK_UNINTERRUPTIBLE);
>
> if (si_mask & SYS_INFO_ALL_BT)
> trigger_all_cpu_backtrace();
>
>- if (si_mask & SYS_INFO_BLOCKED_TASKS)
>- show_state_filter(TASK_UNINTERRUPTIBLE);
>+ if (si_mask & SYS_INFO_TASKS)
>+ show_state();
>+
>+ if (si_mask & SYS_INFO_FTRACE)
>+ ftrace_dump(DUMP_ALL);
> }
>
> void sys_info(unsigned long si_mask)
>
--- Thanks!
https://lore.kernel.org/all/EE579805-42F2-4C58-B752-F28779EEB717@grrlz.net/
© 2016 - 2026 Red Hat, Inc.