tools/workqueue/wq_dump.py | 66 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 3 deletions(-)
Currently, wq_dump.py displays static affinity scopes and pool topology,
offering no visibility into in-flight work items or transient pool
states. Enhance wq_dump.py to:
1. Distinguish between normal ("bh") and high-priority ("bh-hi") BH
worker pools, and report transient pool states such as
"draining" (POOL_BH_DRAINING) or "disassociated"
(POOL_DISASSOCIATED) when an associated CPU is offlined.
2. Provide live busy worker inspection via a new -b|--busy
command-line option. This iterates through pool->busy_hash to
display in-flight workers, identifying their task PID/comm (or
BH context), target workqueue, callback function, in-flight
execution duration, and any custom work item description.
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
Changes since v1:
- Dropped former patch 1/2 ("tools/workqueue/wq_dump.py: Support
backward compatibility for wq->attrs rename") as it has already been
merged upstream
- Corrected POOL_DISASSOCIATED flag handling. Restrict inspection to
per-CPU non-BH worker pools when their associated CPU is offline,
preventing "disassociated" from erroneously appearing on every BH and
unbound pool where the flag persists by design (Tejun Heo)
- Hardened lockless busy_hash traversal against process_one_work()
entry and exit windows (Tejun Heo)
- Preserved backward compatibility for worker->current_start
(Tejun Heo)
- Fixed 32-bit jiffies wrap-around; masked (jiffies - start) to the
target architecture's word size (jiffies_mask) so duration arithmetic
does not yield negative values following INITIAL_JIFFIES overflow
(Tejun Heo)
- Link to v1: https://lore.kernel.org/lkml/20260831181554.117795-1-atomlin@atomlin.com/
---
tools/workqueue/wq_dump.py | 66 ++++++++++++++++++++++++++++++++++++--
1 file changed, 63 insertions(+), 3 deletions(-)
diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py
index 9313ebe0c525..c5659c9d9273 100644
--- a/tools/workqueue/wq_dump.py
+++ b/tools/workqueue/wq_dump.py
@@ -29,6 +29,10 @@ Lists all worker pools indexed by their ID. For each pool:
workers number of all workers
cpu CPU the pool is associated with (per-cpu pool)
cpus CPUs the workers in the pool can run on (unbound pool)
+ flags pool flags (bh, draining, disassociated)
+
+ If -b|--busy is specified, lists all busy workers currently executing
+ work items, their task PID/comm, workqueue, callback function, and duration.
Workqueue CPU -> pool
=====================
@@ -49,12 +53,14 @@ import sys
import argparse
parser = argparse.ArgumentParser(description=desc,
formatter_class=argparse.RawTextHelpFormatter)
+parser.add_argument('-b', '--busy', action='store_true',
+ help='Show busy workers currently executing work items')
args = parser.parse_args()
import drgn
-from drgn.helpers.linux.list import list_for_each_entry,list_empty
+from drgn.helpers.linux.list import list_for_each_entry, list_empty, hlist_for_each_entry
from drgn.helpers.linux.percpu import per_cpu_ptr
-from drgn.helpers.linux.cpumask import for_each_cpu,for_each_possible_cpu
+from drgn.helpers.linux.cpumask import for_each_cpu, for_each_possible_cpu
from drgn.helpers.linux.nodemask import for_each_node
from drgn.helpers.linux.idr import idr_for_each
@@ -62,6 +68,10 @@ def err(s):
print(s, file=sys.stderr, flush=True)
sys.exit(1)
+def get_hz():
+ cs = prog['clocksource_jiffies']
+ return round(1000000000 / (cs.mult.value_() >> cs.shift.value_()))
+
def cpumask_str(cpumask):
output = ""
base = 0
@@ -84,6 +94,12 @@ def wq_attrs(wq):
except AttributeError:
return wq.unbound_attrs
+def worker_current_start(worker):
+ try:
+ return worker.current_start.value_()
+ except AttributeError:
+ return 0
+
def wq_type_str(wq):
if wq.flags & WQ_BH:
return f'{"bh":{wq_type_len}}'
@@ -118,9 +134,14 @@ WQ_AFFN_NUMA = prog['WQ_AFFN_NUMA']
WQ_AFFN_SYSTEM = prog['WQ_AFFN_SYSTEM']
POOL_BH = prog['POOL_BH']
+POOL_BH_DRAINING = prog['POOL_BH_DRAINING']
+POOL_DISASSOCIATED = prog['POOL_DISASSOCIATED']
+HIGHPRI_NICE_LEVEL = prog['HIGHPRI_NICE_LEVEL']
WQ_NAME_LEN = prog['WQ_NAME_LEN'].value_()
cpumask_str_len = len(cpumask_str(wq_unbound_cpumask))
+hz = get_hz()
+jiffies_mask = (1 << (prog['jiffies'].type_.size * 8)) - 1 if 'jiffies' in prog else 0
print('Affinity Scopes')
print('===============')
@@ -168,7 +189,12 @@ for pi, pool in idr_for_each(worker_pool_idr):
if pool.cpu >= 0:
print(f'cpu={pool.cpu.value_():3}', end='')
if pool.flags & POOL_BH:
- print(' bh', end='')
+ bh_type = 'bh-hi' if pool.attrs.nice == HIGHPRI_NICE_LEVEL else 'bh'
+ print(f' {bh_type}', end='')
+ if pool.flags & POOL_BH_DRAINING:
+ print(' draining', end='')
+ elif pool.flags & POOL_DISASSOCIATED:
+ print(' disassociated', end='')
else:
print(f'cpus={cpumask_str(pool.attrs.cpumask)}', end='')
print(f' pod_cpus={cpumask_str(pool.attrs.__pod_cpumask)}', end='')
@@ -176,6 +202,40 @@ for pi, pool in idr_for_each(worker_pool_idr):
print(' strict', end='')
print('')
+ if args.busy:
+ for bkt in pool.busy_hash:
+ for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
+ for _ in range(3):
+ try:
+ pwq = worker.current_pwq
+ func = worker.current_func.value_()
+ if not pwq.value_() or not func:
+ continue
+
+ wq_name = pwq.wq.name.string_().decode()
+ fn_name = prog.symbol(func).name
+
+ dur_str = ''
+ start = worker_current_start(worker)
+ if 'jiffies' in prog and start:
+ jiffies = prog['jiffies'].value_()
+ dur_s = ((jiffies - start) & jiffies_mask) // hz
+ dur_str = f' for {dur_s}s'
+
+ if pool.flags & POOL_BH:
+ w_id = 'bh' if pool.attrs.nice != HIGHPRI_NICE_LEVEL else 'bh-hi'
+ elif worker.task.value_():
+ w_id = f'PID {worker.task.pid.value_():<6} ({worker.task.comm.string_().decode()})'
+ else:
+ w_id = f'worker[{worker.id.value_()}]'
+
+ desc = worker.desc.string_().decode()
+ desc_str = f' desc="{desc}"' if desc and desc != wq_name else ''
+ print(f' busy: {w_id}: {wq_name}:{fn_name}{dur_str}{desc_str}')
+ break
+ except Exception:
+ continue
+
print('')
print('Workqueue CPU -> pool')
print('=====================')
--
2.55.0On Fri, Sep 11, 2026 at 05:06:25PM -0400, Aaron Tomlin wrote:
> @@ -176,6 +202,40 @@ for pi, pool in idr_for_each(worker_pool_idr):
> print(' strict', end='')
> print('')
>
> + if args.busy:
> + for bkt in pool.busy_hash:
> + for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
> + for _ in range(3):
> + try:
...
> + except Exception:
> + continue
Is a bare except the right scope here? It swallows all other silly
exceptions, so a typo anywhere in this block makes -b print nothing at
all instead of failing.
I would go for drgn.FaultError and LookupError, these two that are
actually expected.
--breno
On Mon, Sep 14, 2026 at 01:55:36AM -0700, Breno Leitao wrote:
> On Fri, Sep 11, 2026 at 05:06:25PM -0400, Aaron Tomlin wrote:
> > @@ -176,6 +202,40 @@ for pi, pool in idr_for_each(worker_pool_idr):
> > print(' strict', end='')
> > print('')
> >
> > + if args.busy:
> > + for bkt in pool.busy_hash:
> > + for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
> > + for _ in range(3):
> > + try:
> ...
> > + except Exception:
> > + continue
>
> Is a bare except the right scope here? It swallows all other silly
> exceptions, so a typo anywhere in this block makes -b print nothing at
> all instead of failing.
>
> I would go for drgn.FaultError and LookupError, these two that are
> actually expected.
>
> --breno
Hi Breno,
Thank you for your feedback.
That was an oversight. Indeed, drgn.FaultError (or imported FaultError) and
LookupError (i.e. prog.symbol(0)) is more appropriate.
>>> w = Object(prog, 'struct worker *', address=0x0)
>>> print(w)
Traceback (most recent call last):
File "<python-input-2>", line 1, in <module>
print(w)
~~~~~^^^
_drgn.FaultError: address is not mapped: 0x0
Kind regards,
--
Aaron Tomlin
Hello, Aaron.
On Fri, Sep 11, 2026 at 05:06:25PM -0400, Aaron Tomlin wrote:
> + for bkt in pool.busy_hash:
> + for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
worker->hentry shares storage with the idle-list entry, so a worker going
idle can send this iterator into the circular idle list. Leaving idle can
leave a self-link with WORKER_IDLE clear. The per-worker retries do not
bound either case.
Could you break out of the bucket on an idle worker, a repeated address, or
a traversal read fault? Warn that the pool's busy-worker output is
incomplete and ask the user to retry, then continue with the other pools.
The exception handler needs to cover iterator advancement too.
Thanks.
--
tejun
On Sun, Sep 13, 2026 at 06:01:41AM -1000, Tejun Heo wrote:
> Hello, Aaron.
>
> On Fri, Sep 11, 2026 at 05:06:25PM -0400, Aaron Tomlin wrote:
> > + for bkt in pool.busy_hash:
> > + for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
>
> worker->hentry shares storage with the idle-list entry, so a worker going
> idle can send this iterator into the circular idle list. Leaving idle can
> leave a self-link with WORKER_IDLE clear. The per-worker retries do not
> bound either case.
>
> Could you break out of the bucket on an idle worker, a repeated address, or
> a traversal read fault? Warn that the pool's busy-worker output is
> incomplete and ask the user to retry, then continue with the other pools.
> The exception handler needs to cover iterator advancement too.
>
> Thanks.
>
> --
> tejun
Hi Tejun,
Understood, we need to address the lockless race conditions on a live
system without holding pool->lock. How about the following?
- Guard lockless busy_hash traversal against circular idle list
diversions, self-links, and traversal read faults: check for
WORKER_IDLE, track visited worker addresses per bucket, cover iterator
advancement with (drgn.FaultError, LookupError) exception handling, and
warn if a pool's busy worker dump was incomplete
diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py
index 4a3281bc5605..32be5c69a61b 100644
--- a/tools/workqueue/wq_dump.py
+++ b/tools/workqueue/wq_dump.py
@@ -137,6 +137,7 @@ POOL_BH = prog['POOL_BH']
POOL_BH_DRAINING = prog['POOL_BH_DRAINING']
POOL_DISASSOCIATED = prog['POOL_DISASSOCIATED']
HIGHPRI_NICE_LEVEL = prog['HIGHPRI_NICE_LEVEL']
+WORKER_IDLE = prog['WORKER_IDLE']
WQ_NAME_LEN = prog['WQ_NAME_LEN'].value_()
cpumask_str_len = len(cpumask_str(wq_unbound_cpumask))
@@ -203,38 +204,55 @@ for pi, pool in idr_for_each(worker_pool_idr):
print('')
if args.busy:
+ incomplete = False
for bkt in pool.busy_hash:
- for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
- for _ in range(3):
- try:
- pwq = worker.current_pwq
- func = worker.current_func.value_()
- if not pwq.value_() or not func:
+ if incomplete:
+ break
+ seen = set()
+ try:
+ for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
+ addr = worker.value_()
+ if addr in seen or (worker.flags & WORKER_IDLE):
+ incomplete = True
+ break
+ seen.add(addr)
+
+ for _ in range(3):
+ try:
+ pwq = worker.current_pwq
+ func = worker.current_func.value_()
+ if not pwq.value_() or not func:
+ continue
+
+ wq_name = pwq.wq.name.string_().decode()
+ fn_name = prog.symbol(func).name
+
+ dur_str = ''
+ start = worker_current_start(worker)
+ if 'jiffies' in prog and start:
+ jiffies = prog['jiffies'].value_()
+ dur_s = ((jiffies - start) & jiffies_mask) // hz
+ dur_str = f' for {dur_s}s'
+
+ if pool.flags & POOL_BH:
+ w_id = 'bh' if pool.attrs.nice != HIGHPRI_NICE_LEVEL else 'bh-hi'
+ elif worker.task.value_():
+ w_id = f'PID {worker.task.pid.value_():<6} ({worker.task.comm.string_().decode()})'
+ else:
+ w_id = f'worker[{worker.id.value_()}]'
+
+ desc = worker.desc.string_().decode()
+ desc_str = f' desc="{desc}"' if desc and desc != wq_name else ''
+ print(f' busy: {w_id}: {wq_name}:{fn_name}{dur_str}{desc_str}')
+ break
+ except (drgn.FaultError, LookupError):
continue
+ except (drgn.FaultError, LookupError):
+ incomplete = True
+ break
- wq_name = pwq.wq.name.string_().decode()
- fn_name = prog.symbol(func).name
-
- dur_str = ''
- start = worker_current_start(worker)
- if 'jiffies' in prog and start:
- jiffies = prog['jiffies'].value_()
- dur_s = ((jiffies - start) & jiffies_mask) // hz
- dur_str = f' for {dur_s}s'
-
- if pool.flags & POOL_BH:
- w_id = 'bh' if pool.attrs.nice != HIGHPRI_NICE_LEVEL else 'bh-hi'
- elif worker.task.value_():
- w_id = f'PID {worker.task.pid.value_():<6} ({worker.task.comm.string_().decode()})'
- else:
- w_id = f'worker[{worker.id.value_()}]'
-
- desc = worker.desc.string_().decode()
- desc_str = f' desc="{desc}"' if desc and desc != wq_name else ''
- print(f' busy: {w_id}: {wq_name}:{fn_name}{dur_str}{desc_str}')
- break
- except (drgn.FaultError, LookupError):
- continue
+ if incomplete:
+ print(f' warning: pool[{pi:02}] busy worker dump incomplete, please retry')
print('')
print('Workqueue CPU -> pool')
Kind regards,
--
Aaron Tomlin
© 2016 - 2026 Red Hat, Inc.