include/linux/debug_locks.h | 17 ++++++++++++++++- kernel/hung_task.c | 2 +- kernel/locking/lockdep.c | 14 +++++++++++--- 3 files changed, 28 insertions(+), 5 deletions(-)
Currently, check_hung_uninterruptible_tasks() reports details of locks
held in the system. Also, lockdep_print_held_locks() does not report
details of locks held by a thread if that thread is in TASK_RUNNING state.
Several years of experience of debugging without vmcore tells me that
these limitations have been a barrier for understanding what went wrong
in syzbot's "INFO: task hung in" reports.
I initially thought that the cause of "INFO: task hung in" reports is
due to over-stressing. But I understood that over-stressing is unlikely.
I now consider that there likely is a deadlock/livelock bug where lockdep
cannot report as a deadlock when "INFO: task hung in" is reported.
A typical case is that thread-1 is waiting for something to happen (e.g.
wait_event_*()) with a lock held. When thread-2 tries to hold that lock
using e.g. mutex_lock(), check_hung_uninterruptible_tasks() reports that
thread-2 is hung and thread-1 is holding a lock which thread-2 is trying
to hold. But currently check_hung_uninterruptible_tasks() cannot report
the exact location of thread-1 which gives us an important hint for
understanding why thread-1 is holding that lock for so long period.
When check_hung_uninterruptible_tasks() reports a thread waiting for a
lock, it is important to report backtrace of threads which already held
that lock. Therefore, allow check_hung_uninterruptible_tasks() to report
the exact location of threads which is holding any lock.
To deduplicate code, share debug_show_all_{locks,lock_holders}() using
a flag. As a side effect of sharing, __debug_show_all_locks() skips
current thread if the caller is holding no lock, for reporting RCU lock
taken inside __debug_show_all_locks() is generally useless.
Signed-off-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
---
Changes in v2:
Share debug_show_all_lock_holders() and debug_show_all_locks(),
suggested by Waiman Long <longman@redhat.com>.
include/linux/debug_locks.h | 17 ++++++++++++++++-
kernel/hung_task.c | 2 +-
kernel/locking/lockdep.c | 14 +++++++++++---
3 files changed, 28 insertions(+), 5 deletions(-)
diff --git a/include/linux/debug_locks.h b/include/linux/debug_locks.h
index dbb409d77d4f..b45c89fadfe4 100644
--- a/include/linux/debug_locks.h
+++ b/include/linux/debug_locks.h
@@ -48,7 +48,18 @@ extern int debug_locks_off(void);
#endif
#ifdef CONFIG_LOCKDEP
-extern void debug_show_all_locks(void);
+extern void __debug_show_all_locks(bool show_stack);
+
+static inline void debug_show_all_locks(void)
+{
+ __debug_show_all_locks(false);
+}
+
+static inline void debug_show_all_lock_holders(void)
+{
+ __debug_show_all_locks(true);
+}
+
extern void debug_show_held_locks(struct task_struct *task);
extern void debug_check_no_locks_freed(const void *from, unsigned long len);
extern void debug_check_no_locks_held(void);
@@ -61,6 +72,10 @@ static inline void debug_show_held_locks(struct task_struct *task)
{
}
+static inline void debug_show_all_lock_holders(void)
+{
+}
+
static inline void
debug_check_no_locks_freed(const void *from, unsigned long len)
{
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index bb2354f73ded..18e22bbb714f 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -205,7 +205,7 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
unlock:
rcu_read_unlock();
if (hung_task_show_lock)
- debug_show_all_locks();
+ debug_show_all_lock_holders();
if (hung_task_show_all_bt) {
hung_task_show_all_bt = false;
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 64a13eb56078..7870f7e5c46b 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -55,6 +55,7 @@
#include <linux/rcupdate.h>
#include <linux/kprobes.h>
#include <linux/lockdep.h>
+#include <linux/sched/debug.h>
#include <asm/sections.h>
@@ -6485,7 +6486,7 @@ void debug_check_no_locks_held(void)
EXPORT_SYMBOL_GPL(debug_check_no_locks_held);
#ifdef __KERNEL__
-void debug_show_all_locks(void)
+void __debug_show_all_locks(bool show_stack)
{
struct task_struct *g, *p;
@@ -6493,12 +6494,19 @@ void debug_show_all_locks(void)
pr_warn("INFO: lockdep is turned off.\n");
return;
}
- pr_warn("\nShowing all locks held in the system:\n");
+ if (show_stack)
+ pr_warn("\nShowing all threads with locks held in the system:\n");
+ else
+ pr_warn("\nShowing all locks held in the system:\n");
rcu_read_lock();
for_each_process_thread(g, p) {
if (!p->lockdep_depth)
continue;
+ if (p == current && p->lockdep_depth == 1)
+ continue;
+ if (show_stack)
+ sched_show_task(p);
lockdep_print_held_locks(p);
touch_nmi_watchdog();
touch_all_softlockup_watchdogs();
@@ -6508,7 +6516,7 @@ void debug_show_all_locks(void)
pr_warn("\n");
pr_warn("=============================================\n\n");
}
-EXPORT_SYMBOL_GPL(debug_show_all_locks);
+EXPORT_SYMBOL_GPL(__debug_show_all_locks);
#endif
/*
--
2.18.4On 9/16/22 11:57, Tetsuo Handa wrote:
> Currently, check_hung_uninterruptible_tasks() reports details of locks
> held in the system. Also, lockdep_print_held_locks() does not report
> details of locks held by a thread if that thread is in TASK_RUNNING state.
> Several years of experience of debugging without vmcore tells me that
> these limitations have been a barrier for understanding what went wrong
> in syzbot's "INFO: task hung in" reports.
>
> I initially thought that the cause of "INFO: task hung in" reports is
> due to over-stressing. But I understood that over-stressing is unlikely.
> I now consider that there likely is a deadlock/livelock bug where lockdep
> cannot report as a deadlock when "INFO: task hung in" is reported.
>
> A typical case is that thread-1 is waiting for something to happen (e.g.
> wait_event_*()) with a lock held. When thread-2 tries to hold that lock
> using e.g. mutex_lock(), check_hung_uninterruptible_tasks() reports that
> thread-2 is hung and thread-1 is holding a lock which thread-2 is trying
> to hold. But currently check_hung_uninterruptible_tasks() cannot report
> the exact location of thread-1 which gives us an important hint for
> understanding why thread-1 is holding that lock for so long period.
>
> When check_hung_uninterruptible_tasks() reports a thread waiting for a
> lock, it is important to report backtrace of threads which already held
> that lock. Therefore, allow check_hung_uninterruptible_tasks() to report
> the exact location of threads which is holding any lock.
>
> To deduplicate code, share debug_show_all_{locks,lock_holders}() using
> a flag. As a side effect of sharing, __debug_show_all_locks() skips
> current thread if the caller is holding no lock, for reporting RCU lock
> taken inside __debug_show_all_locks() is generally useless.
>
> Signed-off-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
> ---
> Changes in v2:
> Share debug_show_all_lock_holders() and debug_show_all_locks(),
> suggested by Waiman Long <longman@redhat.com>.
>
> include/linux/debug_locks.h | 17 ++++++++++++++++-
> kernel/hung_task.c | 2 +-
> kernel/locking/lockdep.c | 14 +++++++++++---
> 3 files changed, 28 insertions(+), 5 deletions(-)
>
> diff --git a/include/linux/debug_locks.h b/include/linux/debug_locks.h
> index dbb409d77d4f..b45c89fadfe4 100644
> --- a/include/linux/debug_locks.h
> +++ b/include/linux/debug_locks.h
> @@ -48,7 +48,18 @@ extern int debug_locks_off(void);
> #endif
>
> #ifdef CONFIG_LOCKDEP
> -extern void debug_show_all_locks(void);
> +extern void __debug_show_all_locks(bool show_stack);
> +
> +static inline void debug_show_all_locks(void)
> +{
> + __debug_show_all_locks(false);
> +}
> +
> +static inline void debug_show_all_lock_holders(void)
> +{
> + __debug_show_all_locks(true);
> +}
> +
> extern void debug_show_held_locks(struct task_struct *task);
> extern void debug_check_no_locks_freed(const void *from, unsigned long len);
> extern void debug_check_no_locks_held(void);
> @@ -61,6 +72,10 @@ static inline void debug_show_held_locks(struct task_struct *task)
> {
> }
>
> +static inline void debug_show_all_lock_holders(void)
> +{
> +}
> +
> static inline void
> debug_check_no_locks_freed(const void *from, unsigned long len)
> {
> diff --git a/kernel/hung_task.c b/kernel/hung_task.c
> index bb2354f73ded..18e22bbb714f 100644
> --- a/kernel/hung_task.c
> +++ b/kernel/hung_task.c
> @@ -205,7 +205,7 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
> unlock:
> rcu_read_unlock();
> if (hung_task_show_lock)
> - debug_show_all_locks();
> + debug_show_all_lock_holders();
>
> if (hung_task_show_all_bt) {
> hung_task_show_all_bt = false;
> diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
> index 64a13eb56078..7870f7e5c46b 100644
> --- a/kernel/locking/lockdep.c
> +++ b/kernel/locking/lockdep.c
> @@ -55,6 +55,7 @@
> #include <linux/rcupdate.h>
> #include <linux/kprobes.h>
> #include <linux/lockdep.h>
> +#include <linux/sched/debug.h>
>
> #include <asm/sections.h>
>
> @@ -6485,7 +6486,7 @@ void debug_check_no_locks_held(void)
> EXPORT_SYMBOL_GPL(debug_check_no_locks_held);
>
> #ifdef __KERNEL__
> -void debug_show_all_locks(void)
> +void __debug_show_all_locks(bool show_stack)
> {
> struct task_struct *g, *p;
>
> @@ -6493,12 +6494,19 @@ void debug_show_all_locks(void)
> pr_warn("INFO: lockdep is turned off.\n");
> return;
> }
> - pr_warn("\nShowing all locks held in the system:\n");
> + if (show_stack)
> + pr_warn("\nShowing all threads with locks held in the system:\n");
> + else
> + pr_warn("\nShowing all locks held in the system:\n");
>
> rcu_read_lock();
> for_each_process_thread(g, p) {
> if (!p->lockdep_depth)
> continue;
> + if (p == current && p->lockdep_depth == 1)
> + continue;
> + if (show_stack)
> + sched_show_task(p);
> lockdep_print_held_locks(p);
> touch_nmi_watchdog();
> touch_all_softlockup_watchdogs();
> @@ -6508,7 +6516,7 @@ void debug_show_all_locks(void)
> pr_warn("\n");
> pr_warn("=============================================\n\n");
> }
> -EXPORT_SYMBOL_GPL(debug_show_all_locks);
> +EXPORT_SYMBOL_GPL(__debug_show_all_locks);
> #endif
>
> /*
Acked-by: Waiman Long <longman@redhat.com>
Can this patch go to linux.git ?
On 2022/09/17 3:41, Waiman Long wrote:
> On 9/16/22 11:57, Tetsuo Handa wrote:
>> Currently, check_hung_uninterruptible_tasks() reports details of locks
>> held in the system. Also, lockdep_print_held_locks() does not report
>> details of locks held by a thread if that thread is in TASK_RUNNING state.
>> Several years of experience of debugging without vmcore tells me that
>> these limitations have been a barrier for understanding what went wrong
>> in syzbot's "INFO: task hung in" reports.
>>
>> I initially thought that the cause of "INFO: task hung in" reports is
>> due to over-stressing. But I understood that over-stressing is unlikely.
>> I now consider that there likely is a deadlock/livelock bug where lockdep
>> cannot report as a deadlock when "INFO: task hung in" is reported.
>>
>> A typical case is that thread-1 is waiting for something to happen (e.g.
>> wait_event_*()) with a lock held. When thread-2 tries to hold that lock
>> using e.g. mutex_lock(), check_hung_uninterruptible_tasks() reports that
>> thread-2 is hung and thread-1 is holding a lock which thread-2 is trying
>> to hold. But currently check_hung_uninterruptible_tasks() cannot report
>> the exact location of thread-1 which gives us an important hint for
>> understanding why thread-1 is holding that lock for so long period.
>>
>> When check_hung_uninterruptible_tasks() reports a thread waiting for a
>> lock, it is important to report backtrace of threads which already held
>> that lock. Therefore, allow check_hung_uninterruptible_tasks() to report
>> the exact location of threads which is holding any lock.
>>
>> To deduplicate code, share debug_show_all_{locks,lock_holders}() using
>> a flag. As a side effect of sharing, __debug_show_all_locks() skips
>> current thread if the caller is holding no lock, for reporting RCU lock
>> taken inside __debug_show_all_locks() is generally useless.
>>
>> Signed-off-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
> Acked-by: Waiman Long <longman@redhat.com>
On 10/3/22 18:18, Tetsuo Handa wrote:
> Can this patch go to linux.git ?
It is now up to Peter or Ingo to take it to tip.
Cheers,
Longman
>
> On 2022/09/17 3:41, Waiman Long wrote:
>> On 9/16/22 11:57, Tetsuo Handa wrote:
>>> Currently, check_hung_uninterruptible_tasks() reports details of locks
>>> held in the system. Also, lockdep_print_held_locks() does not report
>>> details of locks held by a thread if that thread is in TASK_RUNNING state.
>>> Several years of experience of debugging without vmcore tells me that
>>> these limitations have been a barrier for understanding what went wrong
>>> in syzbot's "INFO: task hung in" reports.
>>>
>>> I initially thought that the cause of "INFO: task hung in" reports is
>>> due to over-stressing. But I understood that over-stressing is unlikely.
>>> I now consider that there likely is a deadlock/livelock bug where lockdep
>>> cannot report as a deadlock when "INFO: task hung in" is reported.
>>>
>>> A typical case is that thread-1 is waiting for something to happen (e.g.
>>> wait_event_*()) with a lock held. When thread-2 tries to hold that lock
>>> using e.g. mutex_lock(), check_hung_uninterruptible_tasks() reports that
>>> thread-2 is hung and thread-1 is holding a lock which thread-2 is trying
>>> to hold. But currently check_hung_uninterruptible_tasks() cannot report
>>> the exact location of thread-1 which gives us an important hint for
>>> understanding why thread-1 is holding that lock for so long period.
>>>
>>> When check_hung_uninterruptible_tasks() reports a thread waiting for a
>>> lock, it is important to report backtrace of threads which already held
>>> that lock. Therefore, allow check_hung_uninterruptible_tasks() to report
>>> the exact location of threads which is holding any lock.
>>>
>>> To deduplicate code, share debug_show_all_{locks,lock_holders}() using
>>> a flag. As a side effect of sharing, __debug_show_all_locks() skips
>>> current thread if the caller is holding no lock, for reporting RCU lock
>>> taken inside __debug_show_all_locks() is generally useless.
>>>
>>> Signed-off-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
>> Acked-by: Waiman Long <longman@redhat.com>
Peter or Ingo, can you take this patch?
On 2022/10/05 3:09, Waiman Long wrote:
> On 10/3/22 18:18, Tetsuo Handa wrote:
>> Can this patch go to linux.git ?
>
> It is now up to Peter or Ingo to take it to tip.
>
> Cheers,
> Longman
>
>>
>> On 2022/09/17 3:41, Waiman Long wrote:
>>> On 9/16/22 11:57, Tetsuo Handa wrote:
>>>> Currently, check_hung_uninterruptible_tasks() reports details of locks
>>>> held in the system. Also, lockdep_print_held_locks() does not report
>>>> details of locks held by a thread if that thread is in TASK_RUNNING state.
>>>> Several years of experience of debugging without vmcore tells me that
>>>> these limitations have been a barrier for understanding what went wrong
>>>> in syzbot's "INFO: task hung in" reports.
>>>>
>>>> I initially thought that the cause of "INFO: task hung in" reports is
>>>> due to over-stressing. But I understood that over-stressing is unlikely.
>>>> I now consider that there likely is a deadlock/livelock bug where lockdep
>>>> cannot report as a deadlock when "INFO: task hung in" is reported.
>>>>
>>>> A typical case is that thread-1 is waiting for something to happen (e.g.
>>>> wait_event_*()) with a lock held. When thread-2 tries to hold that lock
>>>> using e.g. mutex_lock(), check_hung_uninterruptible_tasks() reports that
>>>> thread-2 is hung and thread-1 is holding a lock which thread-2 is trying
>>>> to hold. But currently check_hung_uninterruptible_tasks() cannot report
>>>> the exact location of thread-1 which gives us an important hint for
>>>> understanding why thread-1 is holding that lock for so long period.
>>>>
>>>> When check_hung_uninterruptible_tasks() reports a thread waiting for a
>>>> lock, it is important to report backtrace of threads which already held
>>>> that lock. Therefore, allow check_hung_uninterruptible_tasks() to report
>>>> the exact location of threads which is holding any lock.
>>>>
>>>> To deduplicate code, share debug_show_all_{locks,lock_holders}() using
>>>> a flag. As a side effect of sharing, __debug_show_all_locks() skips
>>>> current thread if the caller is holding no lock, for reporting RCU lock
>>>> taken inside __debug_show_all_locks() is generally useless.
>>>>
>>>> Signed-off-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
>>> Acked-by: Waiman Long <longman@redhat.com>
>
© 2016 - 2026 Red Hat, Inc.