[PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung

Feng Tang posted 3 patches 1 month, 1 week ago
There is a newer version of this series
[PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung
Posted by Feng Tang 1 month, 1 week ago
When task-hung happens, developers may need different kinds of system
information (call-stacks, memory info, locks, etc.) to help debugging.

Add 'hung_task_sys_info' sysctl knob to take human readable string like
"tasks,mem,timers,locks,ftrace,...", and when task-hung happens, all
requested information will be dumped. (refer kernel/sys_info.c for more
details).

Meanwhile, the newly introduced sys_info() call is used to unify some
existing info-dumping knobs.

Signed-off-by: Feng Tang <feng.tang@linux.alibaba.com>
---
 Documentation/admin-guide/sysctl/kernel.rst |  5 +++
 kernel/hung_task.c                          | 39 +++++++++++++++------
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index a397eeccaea7..45b4408dad31 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -422,6 +422,11 @@ the system boot.
 
 This file shows up if ``CONFIG_DETECT_HUNG_TASK`` is enabled.
 
+hung_task_sys_info
+==================
+A comma separated list of extra system information to be dumped when
+hung task is detected, for example, "tasks,mem,timers,locks,...".
+Refer 'panic_sys_info' section below for more details.
 
 hung_task_timeout_secs
 ======================
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 84b4b049faa5..102be5a8e75a 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -24,6 +24,7 @@
 #include <linux/sched/sysctl.h>
 #include <linux/hung_task.h>
 #include <linux/rwsem.h>
+#include <linux/sys_info.h>
 
 #include <trace/events/sched.h>
 
@@ -60,12 +61,23 @@ static unsigned long __read_mostly sysctl_hung_task_check_interval_secs;
 static int __read_mostly sysctl_hung_task_warnings = 10;
 
 static int __read_mostly did_panic;
-static bool hung_task_show_lock;
 static bool hung_task_call_panic;
-static bool hung_task_show_all_bt;
 
 static struct task_struct *watchdog_task;
 
+/*
+ * A bitmask to control what kinds of system info to be printed when
+ * a hung task is detected, it could be task, memory, lock etc. Refer
+ * include/linux/sys_info.h for detailed bit definition.
+ */
+static unsigned long hung_task_si_mask;
+
+/*
+ * There are several sysctl knobs, and this serves as the runtime
+ * effective sys_info knob
+ */
+static unsigned long cur_si_mask;
+
 #ifdef CONFIG_SMP
 /*
  * Should we dump all CPUs backtraces in a hung task event?
@@ -235,9 +247,10 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
 	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
 	trace_sched_process_hang(t);
 
+	cur_si_mask = hung_task_si_mask;
 	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
 		console_verbose();
-		hung_task_show_lock = true;
+		cur_si_mask |= SYS_INFO_LOCKS;
 		hung_task_call_panic = true;
 	}
 
@@ -260,10 +273,10 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
 			" disables this message.\n");
 		sched_show_task(t);
 		debug_show_blocker(t, timeout);
-		hung_task_show_lock = true;
+		cur_si_mask |= SYS_INFO_LOCKS;
 
 		if (sysctl_hung_task_all_cpu_backtrace)
-			hung_task_show_all_bt = true;
+			cur_si_mask |= SYS_INFO_ALL_BT;
 		if (!sysctl_hung_task_warnings)
 			pr_info("Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings\n");
 	}
@@ -313,7 +326,6 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 	if (test_taint(TAINT_DIE) || did_panic)
 		return;
 
-	hung_task_show_lock = false;
 	rcu_read_lock();
 	for_each_process_thread(g, t) {
 
@@ -329,12 +341,10 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 	}
  unlock:
 	rcu_read_unlock();
-	if (hung_task_show_lock)
-		debug_show_all_locks();
 
-	if (hung_task_show_all_bt) {
-		hung_task_show_all_bt = false;
-		trigger_all_cpu_backtrace();
+	if (unlikely(cur_si_mask)) {
+		sys_info(cur_si_mask);
+		cur_si_mask = 0;
 	}
 
 	if (hung_task_call_panic)
@@ -435,6 +445,13 @@ static const struct ctl_table hung_task_sysctls[] = {
 		.mode		= 0444,
 		.proc_handler	= proc_doulongvec_minmax,
 	},
+	{
+		.procname	= "hung_task_sys_info",
+		.data		= &hung_task_si_mask,
+		.maxlen         = sizeof(hung_task_si_mask),
+		.mode		= 0644,
+		.proc_handler	= sysctl_sys_info_handler,
+	},
 };
 
 static void __init hung_task_sysctl_init(void)
-- 
2.43.5
Re: [PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung
Posted by Petr Mladek 1 month, 1 week ago
On Thu 2025-11-06 10:30:31, Feng Tang wrote:
> When task-hung happens, developers may need different kinds of system
> information (call-stacks, memory info, locks, etc.) to help debugging.
> 
> Add 'hung_task_sys_info' sysctl knob to take human readable string like
> "tasks,mem,timers,locks,ftrace,...", and when task-hung happens, all
> requested information will be dumped. (refer kernel/sys_info.c for more
> details).
> 
> Meanwhile, the newly introduced sys_info() call is used to unify some
> existing info-dumping knobs.
> 
> --- a/kernel/hung_task.c
> +++ b/kernel/hung_task.c
> @@ -60,12 +61,23 @@ static unsigned long __read_mostly sysctl_hung_task_check_interval_secs;
>  static int __read_mostly sysctl_hung_task_warnings = 10;
>  
>  static int __read_mostly did_panic;
> -static bool hung_task_show_lock;
>  static bool hung_task_call_panic;
> -static bool hung_task_show_all_bt;
>  
>  static struct task_struct *watchdog_task;
>  
> +/*
> + * A bitmask to control what kinds of system info to be printed when
> + * a hung task is detected, it could be task, memory, lock etc. Refer
> + * include/linux/sys_info.h for detailed bit definition.
> + */
> +static unsigned long hung_task_si_mask;
> +
> +/*
> + * There are several sysctl knobs, and this serves as the runtime
> + * effective sys_info knob
> + */
> +static unsigned long cur_si_mask;

It seems that this variable is used to pass information between
check_hung_task() and check_hung_uninterruptible_tasks().

And "hung_task_show_lock" and "hung_task_show_all_bt" had the same
purpose.

If I get it correctly, we could move these decisions to
check_hung_uninterruptible_tasks() and avoid the global
variable.

I think that it even makes the code a bit cleaner.

Something like this on top of this patch:

diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 5f0275b2c742..c2a0dfce1e56 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -71,12 +71,6 @@ static struct task_struct *watchdog_task;
  */
 static unsigned long hung_task_si_mask;
 
-/*
- * There are several sysctl knobs, and this serves as the runtime
- * effective sys_info knob
- */
-static unsigned long cur_si_mask;
-
 #ifdef CONFIG_SMP
 /*
  * Should we dump all CPUs backtraces in a hung task event?
@@ -229,11 +223,8 @@ static inline void debug_show_blocker(struct task_struct *task, unsigned long ti
 }
 #endif
 
-static void check_hung_task(struct task_struct *t, unsigned long timeout,
-		unsigned long prev_detect_count)
+static void check_hung_task(struct task_struct *t, unsigned long timeout)
 {
-	unsigned long total_hung_task;
-
 	if (!task_is_hung(t, timeout))
 		return;
 
@@ -243,16 +234,8 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
 	 */
 	sysctl_hung_task_detect_count++;
 
-	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
 	trace_sched_process_hang(t);
 
-	cur_si_mask = hung_task_si_mask;
-	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
-		console_verbose();
-		cur_si_mask |= SYS_INFO_LOCKS;
-		hung_task_call_panic = true;
-	}
-
 	/*
 	 * Ok, the task did not get scheduled for more than 2 minutes,
 	 * complain:
@@ -272,10 +255,7 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
 			" disables this message.\n");
 		sched_show_task(t);
 		debug_show_blocker(t, timeout);
-		cur_si_mask |= SYS_INFO_LOCKS;
 
-		if (sysctl_hung_task_all_cpu_backtrace)
-			cur_si_mask |= SYS_INFO_ALL_BT;
 		if (!sysctl_hung_task_warnings)
 			pr_info("Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings\n");
 	}
@@ -315,8 +295,10 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 {
 	int max_count = sysctl_hung_task_check_count;
 	unsigned long last_break = jiffies;
+	unsigned long total_hung_task;
 	struct task_struct *g, *t;
 	unsigned long prev_detect_count = sysctl_hung_task_detect_count;
+	unsigned long si_mask;
 
 	/*
 	 * If the system crashed already then all bets are off,
@@ -325,6 +307,14 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 	if (test_taint(TAINT_DIE) || did_panic)
 		return;
 
+	si_mask = hung_task_si_mask;
+	if (sysctl_hung_task_warnings || hung_task_call_panic) {
+		si_mask |= SYS_INFO_LOCKS;
+
+		if (sysctl_hung_task_all_cpu_backtrace)
+			si_mask |= SYS_INFO_ALL_BT;
+	}
+
 	rcu_read_lock();
 	for_each_process_thread(g, t) {
 
@@ -336,16 +326,20 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 			last_break = jiffies;
 		}
 
-		check_hung_task(t, timeout, prev_detect_count);
+		check_hung_task(t, timeout);
 	}
  unlock:
 	rcu_read_unlock();
 
-	if (unlikely(cur_si_mask)) {
-		sys_info(cur_si_mask);
-		cur_si_mask = 0;
+	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
+	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
+		console_verbose();
+		hung_task_call_panic = true;
 	}
 
+	if (unlikely(si_mask))
+		sys_info(si_mask);
+
 	if (hung_task_call_panic)
 		panic("hung_task: blocked tasks");
 }

What do you think?

Hmm, maybe, we might still need to pass "prev_detect_count" and
keep "console_verbose()" in check_hung_task().

Best Regards,
Petr
Re: [PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung
Posted by Feng Tang 1 month, 1 week ago
On Mon, Nov 10, 2025 at 06:55:57PM +0100, Petr Mladek wrote:
> On Thu 2025-11-06 10:30:31, Feng Tang wrote:
[...]
> @@ -315,8 +295,10 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
>  {
>  	int max_count = sysctl_hung_task_check_count;
>  	unsigned long last_break = jiffies;
> +	unsigned long total_hung_task;
>  	struct task_struct *g, *t;
>  	unsigned long prev_detect_count = sysctl_hung_task_detect_count;
> +	unsigned long si_mask;
>  
>  	/*
>  	 * If the system crashed already then all bets are off,
> @@ -325,6 +307,14 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
>  	if (test_taint(TAINT_DIE) || did_panic)
>  		return;
>  
> +	si_mask = hung_task_si_mask;
> +	if (sysctl_hung_task_warnings || hung_task_call_panic) {
> +		si_mask |= SYS_INFO_LOCKS;
> +
> +		if (sysctl_hung_task_all_cpu_backtrace)
> +			si_mask |= SYS_INFO_ALL_BT;
> +	}

This probably needs to be moved to after the loop check of
check_hung_task(). 

> +
>  	rcu_read_lock();
>  	for_each_process_thread(g, t) {
>  
> @@ -336,16 +326,20 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
>  			last_break = jiffies;
>  		}
>  
> -		check_hung_task(t, timeout, prev_detect_count);
> +		check_hung_task(t, timeout);
>  	}
>   unlock:
>  	rcu_read_unlock();
>  
> -	if (unlikely(cur_si_mask)) {
> -		sys_info(cur_si_mask);
> -		cur_si_mask = 0;
> +	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
> +	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
> +		console_verbose();
> +		hung_task_call_panic = true;
>  	}
>  
> +	if (unlikely(si_mask))
> +		sys_info(si_mask);
> +
>  	if (hung_task_call_panic)
>  		panic("hung_task: blocked tasks");
>  }
[...]

Thanks,
Feng
Re: [PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung
Posted by Petr Mladek 1 month ago
On Wed 2025-11-12 19:25:27, Feng Tang wrote:
> On Mon, Nov 10, 2025 at 06:55:57PM +0100, Petr Mladek wrote:
> > On Thu 2025-11-06 10:30:31, Feng Tang wrote:
> [...]
> > @@ -315,8 +295,10 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
> >  {
> >  	int max_count = sysctl_hung_task_check_count;
> >  	unsigned long last_break = jiffies;
> > +	unsigned long total_hung_task;
> >  	struct task_struct *g, *t;
> >  	unsigned long prev_detect_count = sysctl_hung_task_detect_count;
> > +	unsigned long si_mask;
> >  
> >  	/*
> >  	 * If the system crashed already then all bets are off,
> > @@ -325,6 +307,14 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
> >  	if (test_taint(TAINT_DIE) || did_panic)
> >  		return;
> >  
> > +	si_mask = hung_task_si_mask;
> > +	if (sysctl_hung_task_warnings || hung_task_call_panic) {
> > +		si_mask |= SYS_INFO_LOCKS;
> > +
> > +		if (sysctl_hung_task_all_cpu_backtrace)
> > +			si_mask |= SYS_INFO_ALL_BT;
> > +	}
> 
> This probably needs to be moved to after the loop check of
> check_hung_task(). 

I did it on purpose because "sysctl_hung_task_warnings" might get
decremented down to "0" in the loop below. But IMHO, we need to print
the information if it was non-zero at the beginning.

It might be worth to add a comment why it has to be done
before the cycle.

> > +
> >  	rcu_read_lock();
> >  	for_each_process_thread(g, t) {
> >  

Best Regards,
Petr
Re: [PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung
Posted by Feng Tang 1 month ago
On Wed, Nov 12, 2025 at 03:44:09PM +0100, Petr Mladek wrote:
> On Wed 2025-11-12 19:25:27, Feng Tang wrote:
> > On Mon, Nov 10, 2025 at 06:55:57PM +0100, Petr Mladek wrote:
> > > On Thu 2025-11-06 10:30:31, Feng Tang wrote:
> > [...]
> > > @@ -315,8 +295,10 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
> > >  {
> > >  	int max_count = sysctl_hung_task_check_count;
> > >  	unsigned long last_break = jiffies;
> > > +	unsigned long total_hung_task;
> > >  	struct task_struct *g, *t;
> > >  	unsigned long prev_detect_count = sysctl_hung_task_detect_count;
> > > +	unsigned long si_mask;
> > >  
> > >  	/*
> > >  	 * If the system crashed already then all bets are off,
> > > @@ -325,6 +307,14 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
> > >  	if (test_taint(TAINT_DIE) || did_panic)
> > >  		return;
> > >  
> > > +	si_mask = hung_task_si_mask;
> > > +	if (sysctl_hung_task_warnings || hung_task_call_panic) {
> > > +		si_mask |= SYS_INFO_LOCKS;
> > > +
> > > +		if (sysctl_hung_task_all_cpu_backtrace)
> > > +			si_mask |= SYS_INFO_ALL_BT;
> > > +	}
> > 
> > This probably needs to be moved to after the loop check of
> > check_hung_task(). 
> 
> I did it on purpose because "sysctl_hung_task_warnings" might get
> decremented down to "0" in the loop below. But IMHO, we need to print
> the information if it was non-zero at the beginning.
> 
> It might be worth to add a comment why it has to be done
> before the cycle.
 
I see your point. Yes, that could happen and should be handled.

My concern was:
1. 'hung_task_call_panic' is actually set in the following loop of 
   checking, and should be checked after the loop
2. when 'sysctl_hung_task_warnings' is not 0 (likely), the
	si_mask |= SYS_INFO_LOCKS
  will make it always call sys_info() will non-zero value, while the
  'hung_task_si_mask' could be pre-set. I just run a simple hung task
  test and can confirm this.

So I made some small changes based on your patches, please help to
review.

diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 5f0275b2c742..5b3a7785d3a2 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -71,12 +71,6 @@ static struct task_struct *watchdog_task;
  */
 static unsigned long hung_task_si_mask;
 
-/*
- * There are several sysctl knobs, and this serves as the runtime
- * effective sys_info knob
- */
-static unsigned long cur_si_mask;
-
 #ifdef CONFIG_SMP
 /*
  * Should we dump all CPUs backtraces in a hung task event?
@@ -229,11 +223,8 @@ static inline void debug_show_blocker(struct task_struct *task, unsigned long ti
 }
 #endif
 
-static void check_hung_task(struct task_struct *t, unsigned long timeout,
-		unsigned long prev_detect_count)
+static void check_hung_task(struct task_struct *t, unsigned long timeout)
 {
-	unsigned long total_hung_task;
-
 	if (!task_is_hung(t, timeout))
 		return;
 
@@ -243,21 +234,13 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
 	 */
 	sysctl_hung_task_detect_count++;
 
-	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
 	trace_sched_process_hang(t);
 
-	cur_si_mask = hung_task_si_mask;
-	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
-		console_verbose();
-		cur_si_mask |= SYS_INFO_LOCKS;
-		hung_task_call_panic = true;
-	}
-
 	/*
 	 * Ok, the task did not get scheduled for more than 2 minutes,
 	 * complain:
 	 */
-	if (sysctl_hung_task_warnings || hung_task_call_panic) {
+	if (sysctl_hung_task_warnings) {
 		if (sysctl_hung_task_warnings > 0)
 			sysctl_hung_task_warnings--;
 		pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
@@ -272,10 +255,7 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
 			" disables this message.\n");
 		sched_show_task(t);
 		debug_show_blocker(t, timeout);
-		cur_si_mask |= SYS_INFO_LOCKS;
 
-		if (sysctl_hung_task_all_cpu_backtrace)
-			cur_si_mask |= SYS_INFO_ALL_BT;
 		if (!sysctl_hung_task_warnings)
 			pr_info("Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings\n");
 	}
@@ -315,8 +295,11 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 {
 	int max_count = sysctl_hung_task_check_count;
 	unsigned long last_break = jiffies;
+	unsigned long total_hung_task;
 	struct task_struct *g, *t;
 	unsigned long prev_detect_count = sysctl_hung_task_detect_count;
+	int need_warning = sysctl_hung_task_warnings;
+	unsigned long si_mask = hung_task_si_mask;
 
 	/*
 	 * If the system crashed already then all bets are off,
@@ -325,6 +308,7 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 	if (test_taint(TAINT_DIE) || did_panic)
 		return;
 
+
 	rcu_read_lock();
 	for_each_process_thread(g, t) {
 
@@ -336,16 +320,29 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 			last_break = jiffies;
 		}
 
-		check_hung_task(t, timeout, prev_detect_count);
+		check_hung_task(t, timeout);
 	}
  unlock:
 	rcu_read_unlock();
 
-	if (unlikely(cur_si_mask)) {
-		sys_info(cur_si_mask);
-		cur_si_mask = 0;
+	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
+	if (!total_hung_task)
+		return;
+
+	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
+		console_verbose();
+		hung_task_call_panic = true;
 	}
 
+	if (need_warning || hung_task_call_panic) {
+		si_mask |= SYS_INFO_LOCKS;
+
+		if (sysctl_hung_task_all_cpu_backtrace)
+			si_mask |= SYS_INFO_ALL_BT;
+	}
+
+	sys_info(si_mask);
+
 	if (hung_task_call_panic)
 		panic("hung_task: blocked tasks");
 }

Thanks,
Feng
Re: [PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung
Posted by Feng Tang 1 month, 1 week ago
Hi Petr,

On Mon, Nov 10, 2025 at 06:55:57PM +0100, Petr Mladek wrote:
> On Thu 2025-11-06 10:30:31, Feng Tang wrote:
> > When task-hung happens, developers may need different kinds of system
> > information (call-stacks, memory info, locks, etc.) to help debugging.
> > 
> > Add 'hung_task_sys_info' sysctl knob to take human readable string like
> > "tasks,mem,timers,locks,ftrace,...", and when task-hung happens, all
> > requested information will be dumped. (refer kernel/sys_info.c for more
> > details).
> > 
> > Meanwhile, the newly introduced sys_info() call is used to unify some
> > existing info-dumping knobs.
> > 
> > --- a/kernel/hung_task.c
> > +++ b/kernel/hung_task.c
> > @@ -60,12 +61,23 @@ static unsigned long __read_mostly sysctl_hung_task_check_interval_secs;
> >  static int __read_mostly sysctl_hung_task_warnings = 10;
> >  
> >  static int __read_mostly did_panic;
> > -static bool hung_task_show_lock;
> >  static bool hung_task_call_panic;
> > -static bool hung_task_show_all_bt;
> >  
> >  static struct task_struct *watchdog_task;
> >  
> > +/*
> > + * A bitmask to control what kinds of system info to be printed when
> > + * a hung task is detected, it could be task, memory, lock etc. Refer
> > + * include/linux/sys_info.h for detailed bit definition.
> > + */
> > +static unsigned long hung_task_si_mask;
> > +
> > +/*
> > + * There are several sysctl knobs, and this serves as the runtime
> > + * effective sys_info knob
> > + */
> > +static unsigned long cur_si_mask;
> 
> It seems that this variable is used to pass information between
> check_hung_task() and check_hung_uninterruptible_tasks().
> 
> And "hung_task_show_lock" and "hung_task_show_all_bt" had the same
> purpose.
> 
> If I get it correctly, we could move these decisions to
> check_hung_uninterruptible_tasks() and avoid the global
> variable.
> 
> I think that it even makes the code a bit cleaner.
> 
> Something like this on top of this patch:
> 
> diff --git a/kernel/hung_task.c b/kernel/hung_task.c
> index 5f0275b2c742..c2a0dfce1e56 100644
> --- a/kernel/hung_task.c
> +++ b/kernel/hung_task.c
> @@ -71,12 +71,6 @@ static struct task_struct *watchdog_task;
>   */
>  static unsigned long hung_task_si_mask;
>  
> -/*
> - * There are several sysctl knobs, and this serves as the runtime
> - * effective sys_info knob
> - */
> -static unsigned long cur_si_mask;
> -
>  #ifdef CONFIG_SMP
>  /*
>   * Should we dump all CPUs backtraces in a hung task event?
> @@ -229,11 +223,8 @@ static inline void debug_show_blocker(struct task_struct *task, unsigned long ti
>  }
>  #endif
>  
> -static void check_hung_task(struct task_struct *t, unsigned long timeout,
> -		unsigned long prev_detect_count)
> +static void check_hung_task(struct task_struct *t, unsigned long timeout)
>  {
> -	unsigned long total_hung_task;
> -
>  	if (!task_is_hung(t, timeout))
>  		return;
>  
> @@ -243,16 +234,8 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
>  	 */
>  	sysctl_hung_task_detect_count++;
>  
> -	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
>  	trace_sched_process_hang(t);
>  
> -	cur_si_mask = hung_task_si_mask;
> -	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
> -		console_verbose();
> -		cur_si_mask |= SYS_INFO_LOCKS;
> -		hung_task_call_panic = true;
> -	}
> -
>  	/*
>  	 * Ok, the task did not get scheduled for more than 2 minutes,
>  	 * complain:
> @@ -272,10 +255,7 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
>  			" disables this message.\n");
>  		sched_show_task(t);
>  		debug_show_blocker(t, timeout);
> -		cur_si_mask |= SYS_INFO_LOCKS;
>  
> -		if (sysctl_hung_task_all_cpu_backtrace)
> -			cur_si_mask |= SYS_INFO_ALL_BT;
>  		if (!sysctl_hung_task_warnings)
>  			pr_info("Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings\n");
>  	}
> @@ -315,8 +295,10 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
>  {
>  	int max_count = sysctl_hung_task_check_count;
>  	unsigned long last_break = jiffies;
> +	unsigned long total_hung_task;
>  	struct task_struct *g, *t;
>  	unsigned long prev_detect_count = sysctl_hung_task_detect_count;
> +	unsigned long si_mask;
>  
>  	/*
>  	 * If the system crashed already then all bets are off,
> @@ -325,6 +307,14 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
>  	if (test_taint(TAINT_DIE) || did_panic)
>  		return;
>  
> +	si_mask = hung_task_si_mask;
> +	if (sysctl_hung_task_warnings || hung_task_call_panic) {
> +		si_mask |= SYS_INFO_LOCKS;
> +
> +		if (sysctl_hung_task_all_cpu_backtrace)
> +			si_mask |= SYS_INFO_ALL_BT;
> +	}
> +
>  	rcu_read_lock();
>  	for_each_process_thread(g, t) {
>  
> @@ -336,16 +326,20 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
>  			last_break = jiffies;
>  		}
>  
> -		check_hung_task(t, timeout, prev_detect_count);
> +		check_hung_task(t, timeout);
>  	}
>   unlock:
>  	rcu_read_unlock();
>  
> -	if (unlikely(cur_si_mask)) {
> -		sys_info(cur_si_mask);
> -		cur_si_mask = 0;
> +	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
> +	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
> +		console_verbose();
> +		hung_task_call_panic = true;
>  	}
>  
> +	if (unlikely(si_mask))
> +		sys_info(si_mask);
> +
>  	if (hung_task_call_panic)
>  		panic("hung_task: blocked tasks");
>  }
> 
> What do you think?

The cleanup looks great to me.

> Hmm, maybe, we might still need to pass "prev_detect_count" and
> keep "console_verbose()" in check_hung_task().

I think moving the console_verbose() here is fine, as the msg printing
in check_hung_task() is mostly pr_err() and pr_info() already.

Thanks,
Feng

> 
> Best Regards,
> Petr
Re: [PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung
Posted by Lance Yang 1 month, 1 week ago

On 2025/11/6 10:30, Feng Tang wrote:
> When task-hung happens, developers may need different kinds of system
> information (call-stacks, memory info, locks, etc.) to help debugging.
> 
> Add 'hung_task_sys_info' sysctl knob to take human readable string like
> "tasks,mem,timers,locks,ftrace,...", and when task-hung happens, all
> requested information will be dumped. (refer kernel/sys_info.c for more
> details).
> 
> Meanwhile, the newly introduced sys_info() call is used to unify some
> existing info-dumping knobs.

Thanks! Just one nit below.

> 
> Signed-off-by: Feng Tang <feng.tang@linux.alibaba.com>
> ---
>   Documentation/admin-guide/sysctl/kernel.rst |  5 +++
>   kernel/hung_task.c                          | 39 +++++++++++++++------
>   2 files changed, 33 insertions(+), 11 deletions(-)
> 
> diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
> index a397eeccaea7..45b4408dad31 100644
> --- a/Documentation/admin-guide/sysctl/kernel.rst
> +++ b/Documentation/admin-guide/sysctl/kernel.rst
> @@ -422,6 +422,11 @@ the system boot.
>   
>   This file shows up if ``CONFIG_DETECT_HUNG_TASK`` is enabled.
>   
> +hung_task_sys_info
> +==================
> +A comma separated list of extra system information to be dumped when
> +hung task is detected, for example, "tasks,mem,timers,locks,...".
> +Refer 'panic_sys_info' section below for more details.
>   
>   hung_task_timeout_secs
>   ======================
> diff --git a/kernel/hung_task.c b/kernel/hung_task.c
> index 84b4b049faa5..102be5a8e75a 100644
> --- a/kernel/hung_task.c
> +++ b/kernel/hung_task.c
> @@ -24,6 +24,7 @@
>   #include <linux/sched/sysctl.h>
>   #include <linux/hung_task.h>
>   #include <linux/rwsem.h>
> +#include <linux/sys_info.h>
>   
>   #include <trace/events/sched.h>
>   
> @@ -60,12 +61,23 @@ static unsigned long __read_mostly sysctl_hung_task_check_interval_secs;
>   static int __read_mostly sysctl_hung_task_warnings = 10;
>   
>   static int __read_mostly did_panic;
> -static bool hung_task_show_lock;
>   static bool hung_task_call_panic;
> -static bool hung_task_show_all_bt;
>   
>   static struct task_struct *watchdog_task;
>   
> +/*
> + * A bitmask to control what kinds of system info to be printed when
> + * a hung task is detected, it could be task, memory, lock etc. Refer
> + * include/linux/sys_info.h for detailed bit definition.
> + */
> +static unsigned long hung_task_si_mask;
> +
> +/*
> + * There are several sysctl knobs, and this serves as the runtime
> + * effective sys_info knob
> + */

Nit: let's make the comment for cur_si_mask even more explicit.
+/*
+ * The effective sys_info mask for the current detection cycle. It
+ * aggregates the base hung_task_si_mask and any flags triggered
+ * by other conditions within this cycle. It is cleared after use.
+ */
> +static unsigned long cur_si_mask;

That makes its lifecycle (aggregate, use, and clear) super obvious ;)

With that, LGTM!

Reviewed-by: Lance Yang <lance.yang@linux.dev>

[...]
Re: [PATCH 2/3] hung_task: Add hung_task_sys_info sysctl to dump sys info on task-hung
Posted by Feng Tang 1 month, 1 week ago
On Thu, Nov 06, 2025 at 11:28:12AM +0800, Lance Yang wrote:
> 
> 
> On 2025/11/6 10:30, Feng Tang wrote:
> > When task-hung happens, developers may need different kinds of system
> > information (call-stacks, memory info, locks, etc.) to help debugging.
> > 
> > Add 'hung_task_sys_info' sysctl knob to take human readable string like
> > "tasks,mem,timers,locks,ftrace,...", and when task-hung happens, all
> > requested information will be dumped. (refer kernel/sys_info.c for more
> > details).
> > 
> > Meanwhile, the newly introduced sys_info() call is used to unify some
> > existing info-dumping knobs.
> 
> Thanks! Just one nit below.
> 
> > 
> > Signed-off-by: Feng Tang <feng.tang@linux.alibaba.com>
> > ---
> >   Documentation/admin-guide/sysctl/kernel.rst |  5 +++
> >   kernel/hung_task.c                          | 39 +++++++++++++++------
> >   2 files changed, 33 insertions(+), 11 deletions(-)
> > 
> > diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
> > index a397eeccaea7..45b4408dad31 100644
> > --- a/Documentation/admin-guide/sysctl/kernel.rst
> > +++ b/Documentation/admin-guide/sysctl/kernel.rst
> > @@ -422,6 +422,11 @@ the system boot.
> >   This file shows up if ``CONFIG_DETECT_HUNG_TASK`` is enabled.
> > +hung_task_sys_info
> > +==================
> > +A comma separated list of extra system information to be dumped when
> > +hung task is detected, for example, "tasks,mem,timers,locks,...".
> > +Refer 'panic_sys_info' section below for more details.
> >   hung_task_timeout_secs
> >   ======================
> > diff --git a/kernel/hung_task.c b/kernel/hung_task.c
> > index 84b4b049faa5..102be5a8e75a 100644
> > --- a/kernel/hung_task.c
> > +++ b/kernel/hung_task.c
> > @@ -24,6 +24,7 @@
> >   #include <linux/sched/sysctl.h>
> >   #include <linux/hung_task.h>
> >   #include <linux/rwsem.h>
> > +#include <linux/sys_info.h>
> >   #include <trace/events/sched.h>
> > @@ -60,12 +61,23 @@ static unsigned long __read_mostly sysctl_hung_task_check_interval_secs;
> >   static int __read_mostly sysctl_hung_task_warnings = 10;
> >   static int __read_mostly did_panic;
> > -static bool hung_task_show_lock;
> >   static bool hung_task_call_panic;
> > -static bool hung_task_show_all_bt;
> >   static struct task_struct *watchdog_task;
> > +/*
> > + * A bitmask to control what kinds of system info to be printed when
> > + * a hung task is detected, it could be task, memory, lock etc. Refer
> > + * include/linux/sys_info.h for detailed bit definition.
> > + */
> > +static unsigned long hung_task_si_mask;
> > +
> > +/*
> > + * There are several sysctl knobs, and this serves as the runtime
> > + * effective sys_info knob
> > + */
> 
> Nit: let's make the comment for cur_si_mask even more explicit.
> +/*
> + * The effective sys_info mask for the current detection cycle. It
> + * aggregates the base hung_task_si_mask and any flags triggered
> + * by other conditions within this cycle. It is cleared after use.
> + */
> > +static unsigned long cur_si_mask;
> 
> That makes its lifecycle (aggregate, use, and clear) super obvious ;)

Yep. Thanks for the imporovement! Will take.

> With that, LGTM!
> 
> Reviewed-by: Lance Yang <lance.yang@linux.dev>
Thanks!

- Feng