include/linux/task_work.h | 3 +++ io_uring/io-wq.c | 23 +++++++++++------- kernel/task_work.c | 51 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-)
We hit hard-lockup reports from iou-wrk threads stuck in
task_work_cancel_match() during io-wq teardown in syzkaller test.
The root cause is that teardown repeatedly rescans the submitter task's
full task_work list under pi_lock, once per matched item.
Two spots are problematic:
1) io_wq_cancel_tw_create() loops calling task_work_cancel_match() to
remove worker-creation callbacks one at a time. Each call re-walks
the entire list from scratch while holding pi_lock.
2) io_worker_exit() unconditionally scans the submitter task_work list
for its own create_work, even when it never queued one. With many
workers exiting simultaneously against a large unrelated task_work
list, this adds up fast.
Fix (1) by adding task_work_cancel_match_all() that unlinks all matching
callbacks in a single traversal, then iterating the returned list locally.
Same try_cmpxchg() synchronisation as before, stops at the work_exited
sentinel.
Fix (2) by skipping the cancel entirely unless create_state indicates a
pending create_work. Since create_state is exclusively owned via
test_and_set_bit_lock, at most one callback can be queued per worker, so
the cancel is also simplified from a loop to a single call.
With this fix the reproducer (FIFO-open + MSG_RING SEND_FD stress) no
longer triggers hard-lockup reports, and task_work_cancel_match samples
drop to microseconds.
Fixes: c80ca4707d1a ("io-wq: cancel task_work on exit only targeting the current 'wq'")
Fixes: 1d5f5ea7cb7d ("io-wq: remove worker to owner tw dependency")
Signed-off-by: Fengnan Chang <changfengnan@bytedance.com>
---
include/linux/task_work.h | 3 +++
io_uring/io-wq.c | 23 +++++++++++-------
kernel/task_work.c | 51 +++++++++++++++++++++++++++++++++++++++
3 files changed, 68 insertions(+), 9 deletions(-)
diff --git a/include/linux/task_work.h b/include/linux/task_work.h
index 0646804860ff1..fb39d18c7c1fe 100644
--- a/include/linux/task_work.h
+++ b/include/linux/task_work.h
@@ -31,6 +31,9 @@ int task_work_add(struct task_struct *task, struct callback_head *twork,
struct callback_head *task_work_cancel_match(struct task_struct *task,
bool (*match)(struct callback_head *, void *data), void *data);
+struct callback_head *
+task_work_cancel_match_all(struct task_struct *task,
+ bool (*match)(struct callback_head *, void *data), void *data);
struct callback_head *task_work_cancel_func(struct task_struct *, task_work_func_t);
bool task_work_cancel(struct task_struct *task, struct callback_head *cb);
void task_work_run(void);
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index 7a9f94a0ce6f2..58144bd5891fa 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -234,13 +234,15 @@ static void io_worker_exit(struct io_worker *worker)
struct io_wq *wq = worker->wq;
struct io_wq_acct *acct = io_wq_get_acct(worker);
- while (1) {
- struct callback_head *cb = task_work_cancel_match(wq->task,
- io_task_worker_match, worker);
-
- if (!cb)
- break;
- io_worker_cancel_cb(worker);
+ if (test_bit(0, &worker->create_state)) {
+ /*
+ * create_state is exclusively owned via test_and_set_bit_lock,
+ * so at most one create_work can be pending per worker — a
+ * single cancel attempt is sufficient here.
+ */
+ if (task_work_cancel_match(wq->task, io_task_worker_match,
+ worker))
+ io_worker_cancel_cb(worker);
}
io_worker_release(worker);
@@ -1319,11 +1321,13 @@ void io_wq_exit_start(struct io_wq *wq)
static void io_wq_cancel_tw_create(struct io_wq *wq)
{
- struct callback_head *cb;
+ struct callback_head *cb, *next;
- while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) {
+ cb = task_work_cancel_match_all(wq->task, io_task_work_match, wq);
+ while (cb) {
struct io_worker *worker;
+ next = cb->next;
worker = container_of(cb, struct io_worker, create_work);
io_worker_cancel_cb(worker);
/*
@@ -1332,6 +1336,7 @@ static void io_wq_cancel_tw_create(struct io_wq *wq)
*/
if (cb->func == create_worker_cont)
kfree(worker);
+ cb = next;
}
}
diff --git a/kernel/task_work.c b/kernel/task_work.c
index 0f7519f8e7c93..c133f6988e844 100644
--- a/kernel/task_work.c
+++ b/kernel/task_work.c
@@ -143,6 +143,57 @@ task_work_cancel_match(struct task_struct *task,
return work;
}
+/**
+ * task_work_cancel_match_all - cancel all pending works matching @match
+ * @task: the task which should execute the work
+ * @match: match function to call
+ * @data: data to be passed in to match function
+ *
+ * Removes all currently queued matching works in one traversal. The returned
+ * callbacks are linked through ->next in their original queue order. This is
+ * useful for teardown paths that need to cancel many callbacks of the same
+ * class without repeatedly rescanning the whole task_work list under
+ * task->pi_lock.
+ *
+ * RETURNS:
+ * The first found work or NULL if not found.
+ */
+struct callback_head *
+task_work_cancel_match_all(struct task_struct *task,
+ bool (*match)(struct callback_head *, void *data),
+ void *data)
+{
+ struct callback_head **pprev = &task->task_works;
+ struct callback_head *work, *next;
+ struct callback_head *head = NULL, **tail = &head;
+ unsigned long flags;
+
+ if (likely(!task_work_pending(task)))
+ return NULL;
+
+ raw_spin_lock_irqsave(&task->pi_lock, flags);
+ work = READ_ONCE(*pprev);
+ while (work && work != &work_exited) {
+ next = READ_ONCE(work->next);
+ if (!match(work, data)) {
+ pprev = &work->next;
+ work = next;
+ continue;
+ }
+
+ if (!try_cmpxchg(pprev, &work, next))
+ continue;
+
+ work->next = NULL;
+ *tail = work;
+ tail = &work->next;
+ work = next;
+ }
+ raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+
+ return head;
+}
+
static bool task_work_func_match(struct callback_head *cb, void *data)
{
return cb->func == data;
--
2.39.5 (Apple Git-154)On 5/19/26 9:12 PM, Fengnan Chang wrote: > We hit hard-lockup reports from iou-wrk threads stuck in > task_work_cancel_match() during io-wq teardown in syzkaller test. > The root cause is that teardown repeatedly rescans the submitter task's > full task_work list under pi_lock, once per matched item. > > Two spots are problematic: > > 1) io_wq_cancel_tw_create() loops calling task_work_cancel_match() to > remove worker-creation callbacks one at a time. Each call re-walks > the entire list from scratch while holding pi_lock. > > 2) io_worker_exit() unconditionally scans the submitter task_work list > for its own create_work, even when it never queued one. With many > workers exiting simultaneously against a large unrelated task_work > list, this adds up fast. > > Fix (1) by adding task_work_cancel_match_all() that unlinks all matching > callbacks in a single traversal, then iterating the returned list locally. > Same try_cmpxchg() synchronisation as before, stops at the work_exited > sentinel. > > Fix (2) by skipping the cancel entirely unless create_state indicates a > pending create_work. Since create_state is exclusively owned via > test_and_set_bit_lock, at most one callback can be queued per worker, so > the cancel is also simplified from a loop to a single call. > > With this fix the reproducer (FIFO-open + MSG_RING SEND_FD stress) no > longer triggers hard-lockup reports, and task_work_cancel_match samples > drop to microseconds. Looks good to me, nicer way to do this too. -- Jens Axboe
"Fengnan Chang" <changfengnan@bytedance.com> writes:
> We hit hard-lockup reports from iou-wrk threads stuck in
It seems like a soft-lockup instead no? From your description,
eventually it solves itself, the task is just uninterruptible while
contending on the spinlock.
> + */
> +struct callback_head *
> +task_work_cancel_match_all(struct task_struct *task,
> + bool (*match)(struct callback_head *, void *data),
> + void *data)
> +{
> + struct callback_head **pprev = &task->task_works;
> + struct callback_head *work, *next;
> + struct callback_head *head = NULL, **tail = &head;
> + unsigned long flags;
> +
> + if (likely(!task_work_pending(task)))
> + return NULL;
> +
> + raw_spin_lock_irqsave(&task->pi_lock, flags);
> + work = READ_ONCE(*pprev);
> + while (work && work != &work_exited) {
> + next = READ_ONCE(work->next);
> + if (!match(work, data)) {
> + pprev = &work->next;
> + work = next;
> + continue;
> + }
> +
> + if (!try_cmpxchg(pprev, &work, next))
> + continue;
IIUC, you could ignore the cmpxchg here because the following loop
iteration on the caller would catch it and retry. In this case, it no
retry in io_wq_cancel_tw_create, which looks weird. Did I miss something?
> +
> + work->next = NULL;
> + *tail = work;
> + tail = &work->next;
> + work = next;
> + }
> + raw_spin_unlock_irqrestore(&task->pi_lock, flags);
> +
> + return head;
> +}
> +
> static bool task_work_func_match(struct callback_head *cb, void *data)
> {
> return cb->func == data;
--
Gabriel Krisman Bertazi
> From: "Gabriel Krisman Bertazi"<krisman@suse.de>
> Date: Wed, May 20, 2026, 21:45
> Subject: Re: [PATCH] io_uring/io-wq: avoid repeated task_work scans during teardown
> To: "Fengnan Chang"<changfengnan@bytedance.com>
> Cc: <axboe@kernel.dk>, <io-uring@vger.kernel.org>, <linux-kernel@vger.kernel.org>, <peterz@infradead.org>, <rostedt@goodmis.org>
> "Fengnan Chang" <changfengnan@bytedance.com> writes:
>
> > We hit hard-lockup reports from iou-wrk threads stuck in
>
> It seems like a soft-lockup instead no? From your description,
> eventually it solves itself, the task is just uninterruptible while
> contending on the spinlock.
hard-lockup, here is the log:
May 19 11:20:41 n154-134-017 kernel: [ 672.229430][ C138] watchdog: CPU138: Watchdog detected hard LOCKUP on cpu 138
May 19 11:20:41 n154-134-017 kernel: [ 672.229435][ C138] Modules linked in: binfmt_misc(E) msr(E) nft_compat(E) x_tables(E) ip_set_hash_net(E) ip_set(E) nf_tables(E) nfnetlink(E) bonding(E) i10nm_edac(E) skx_edac_common(E) nfit(E) edac_core(E) intel_rapl_msr(E) intel_rapl_common(E) intel_uncore_frequency(E) intel_uncore_frequency_common(E) x86_pkg_temp_thermal(E) intel_powerclamp(E) coretemp(E) btrfs(E) ast(E) qat_4xxx(E) snd_pcm(E) kvm_intel(E) drm_client_lib(E) libblake2b(E) tpm_tis(E) snd_timer(E) drm_shmem_helper(E) tpm_tis_core(E) intel_qat(E) snd(E) cxl_acpi(E) pmt_telemetry(E) kvm(E) crc8(E) drm_kms_helper(E) iTCO_wdt(E) tpm(E) raid6_pq(E) cxl_pmem(E) soundcore(E) isst_if_mmio(E) pmt_discovery(E) isst_if_mbox_pci(E) authenc(E) irqbypass(E) aesni_intel(E) gf128mul(E) rapl(E) intel_cstate(E) intel_uncore(E) libnvdimm(E) mei_me(E) pmt_class(E) rng_core(E) dax_hmem(E) pcspkr(E) efi_pstore(E) drm(E) zstd_compress(E) xor(E) i2c_i801(E) idxd(E) bnxt_en(E) mei(E) isst_if_common(E) intel_vsec(E) idxd_bus(E) i2c_smbus(E) i2c_ismt(E) wmi(E) aead(E) i2c_algo_bit(E)
May 19 11:20:41 n154-134-017 kernel: [ 672.229527][ C138] acpi_power_meter(E) button(E) joydev(E) evdev(E) hid_generic(E) usbhid(E) hid(E) acpi_ipmi(E) ipmi_si(E) ipmi_devintf(E) ipmi_msghandler(E) efivarfs(E) autofs4(E) xhci_pci(E) xhci_hcd(E) nvme(E) usbcore(E) usb_common(E) nvme_core(E)
May 19 11:20:41 n154-134-017 kernel: [ 672.229553][ C138] irq event stamp: 47578
May 19 11:20:41 n154-134-017 kernel: [ 672.229555][ C138] hardirqs last enabled at (47577): [<ffffffffa4c49fc9>] _raw_spin_unlock_irqrestore+0x39/0x60
May 19 11:20:41 n154-134-017 kernel: [ 672.229561][ C138] hardirqs last disabled at (47578): [<ffffffffa4c49cd7>] _raw_spin_lock_irqsave+0x67/0x70
May 19 11:20:41 n154-134-017 kernel: [ 672.229566][ C138] softirqs last enabled at (45744): [<ffffffffa21f88c7>] handle_softirqs+0x577/0x840
May 19 11:20:41 n154-134-017 kernel: [ 672.229571][ C138] softirqs last disabled at (45739): [<ffffffffa21f9726>] irq_exit_rcu+0xe6/0x280
May 19 11:20:41 n154-134-017 kernel: [ 672.229576][ C138] CPU: 138 UID: 0 PID: 34918 Comm: iowq-exit-stres Kdump: loaded Tainted: G S EL 7.1.0-rc3-debug-iomap+ #99 PREEMPT(lazy)
May 19 11:20:41 n154-134-017 kernel: [ 672.229583][ C138] Tainted: [S]=CPU_OUT_OF_SPEC, [E]=UNSIGNED_MODULE, [L]=SOFTLOCKUP
May 19 11:20:41 n154-134-017 kernel: [ 672.229585][ C138] Hardware name: Inventec G220-B6/Yichun MLB, BIOS 03.01.02.04.02 10/30/2023
May 19 11:20:41 n154-134-017 kernel: [ 672.229587][ C138] RIP: 0010:native_queued_spin_lock_slowpath+0x55d/0xc90
May 19 11:20:41 n154-134-017 kernel: [ 672.229592][ C138] Code: c0 75 3f 48 8b 8d 30 ff ff ff 48 b8 00 00 00 00 00 fc ff df 48 89 ca 83 e1 07 48 c1 ea 03 49 89 cd 48 01 c2 41 83 c5 03 f3 90 <0f> b6 02 41 38 c5 7c 08 84 c0 0f 85 9c 05 00 00 41 8b 47 08 85 c0
May 19 11:20:41 n154-134-017 kernel: [ 672.229596][ C138] RSP: 0018:ff110085599f7598 EFLAGS: 00000046
May 19 11:20:41 n154-134-017 kernel: [ 672.229599][ C138] RAX: 0000000000000000 RBX: ff110085599f7658 RCX: 0000000000000000
May 19 11:20:41 n154-134-017 kernel: [ 672.229602][ C138] RDX: ffe21c0bcffe7ab9 RSI: 1ffffffff4b0d54e RDI: ffffffffa586aa70
May 19 11:20:41 n154-134-017 kernel: [ 672.229604][ C138] RBP: ff110085599f7680 R08: ffe21c10ab340157 R09: ffe21c10ab340157
May 19 11:20:41 n154-134-017 kernel: [ 672.229607][ C138] R10: ffe21c10ab340156 R11: ff11008559a00ab3 R12: 00000000022c0000
May 19 11:20:41 n154-134-017 kernel: [ 672.229610][ C138] R13: 0000000000000003 R14: ff11008559a00ab0 R15: ff11005e7ff3d5c0
May 19 11:20:41 n154-134-017 kernel: [ 672.229613][ C138] FS: 00007fa81d48e500(0000) GS:ff11005ed9544000(0000) knlGS:0000000000000000
May 19 11:20:41 n154-134-017 kernel: [ 672.229615][ C138] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
May 19 11:20:41 n154-134-017 kernel: [ 672.229618][ C138] CR2: 0000000032f88000 CR3: 0000008432d9f003 CR4: 0000000000f73ef0
May 19 11:20:41 n154-134-017 kernel: [ 672.229621][ C138] PKRU: 55555554
May 19 11:20:41 n154-134-017 kernel: [ 672.229622][ C138] Call Trace:
May 19 11:20:41 n154-134-017 kernel: [ 672.229624][ C138] <TASK>
May 19 11:20:41 n154-134-017 kernel: [ 672.229627][ C138] ? __pfx_native_queued_spin_lock_slowpath+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229632][ C138] ? ring_buffer_unlock_commit+0x130/0x570
May 19 11:20:41 n154-134-017 kernel: [ 672.229638][ C138] ? io_worker_cancel_cb+0x100/0x100
May 19 11:20:41 n154-134-017 kernel: [ 672.229647][ C138] do_raw_spin_lock+0x1e5/0x2a0
May 19 11:20:41 n154-134-017 kernel: [ 672.229652][ C138] ? __pfx_do_raw_spin_lock+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229656][ C138] ? __pfx_function_trace_call+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229661][ C138] ? _raw_spin_lock_irqsave+0x67/0x70
May 19 11:20:41 n154-134-017 kernel: [ 672.229666][ C138] ? __pfx_io_task_work_match+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229671][ C138] _raw_spin_lock_irqsave+0x52/0x70
May 19 11:20:41 n154-134-017 kernel: [ 672.229674][ C138] ? task_work_cancel_match+0xf4/0x260
May 19 11:20:41 n154-134-017 kernel: [ 672.229680][ C138] task_work_cancel_match+0xf4/0x260
May 19 11:20:41 n154-134-017 kernel: [ 672.229686][ C138] ? __pfx_task_work_cancel_match+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229691][ C138] ? __pfx_io_task_work_match+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229696][ C138] ? task_work_cancel_match+0x9/0x260
May 19 11:20:41 n154-134-017 kernel: [ 672.229700][ C138] ? find_held_lock+0x35/0xb0
May 19 11:20:41 n154-134-017 kernel: [ 672.229707][ C138] io_wq_cancel_tw_create+0x82/0xc0
May 19 11:20:41 n154-134-017 kernel: [ 672.229713][ C138] io_wq_put_and_exit+0xdd/0x770
May 19 11:20:41 n154-134-017 kernel: [ 672.229717][ C138] ? xa_find_after+0x1c1/0x330
May 19 11:20:41 n154-134-017 kernel: [ 672.229723][ C138] ? __pfx_io_wq_put_and_exit+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229729][ C138] ? io_uring_del_tctx_node+0x2ce/0x3c0
May 19 11:20:41 n154-134-017 kernel: [ 672.229737][ C138] io_uring_clean_tctx+0x125/0x1b0
May 19 11:20:41 n154-134-017 kernel: [ 672.229742][ C138] ? __pfx_io_uring_clean_tctx+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229746][ C138] ? lock_is_held_type+0xa7/0x120
May 19 11:20:41 n154-134-017 kernel: [ 672.229751][ C138] ? __kasan_check_write+0x18/0x20
May 19 11:20:41 n154-134-017 kernel: [ 672.229758][ C138] io_uring_cancel_generic+0x5cb/0xe70
May 19 11:20:41 n154-134-017 kernel: [ 672.229764][ C138] ? __lock_acquire+0xc71/0x1ea0
May 19 11:20:41 n154-134-017 kernel: [ 672.229769][ C138] ? __pfx_io_uring_cancel_generic+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229776][ C138] ? __kasan_check_write+0x18/0x20
May 19 11:20:41 n154-134-017 kernel: [ 672.229780][ C138] ? do_raw_spin_lock+0x130/0x2a0
May 19 11:20:41 n154-134-017 kernel: [ 672.229785][ C138] ? __pfx_autoremove_wake_function+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229791][ C138] ? _raw_spin_unlock_irq+0x2b/0x50
May 19 11:20:41 n154-134-017 kernel: [ 672.229794][ C138] ? trace_hardirqs_on+0x2e/0x1a0
May 19 11:20:41 n154-134-017 kernel: [ 672.229800][ C138] __io_uring_cancel+0x1f/0x30
May 19 11:20:41 n154-134-017 kernel: [ 672.229804][ C138] do_exit+0x366/0x34e0
May 19 11:20:41 n154-134-017 kernel: [ 672.229810][ C138] ? lock_is_held_type+0xa7/0x120
May 19 11:20:41 n154-134-017 kernel: [ 672.229814][ C138] ? lock_is_held_type+0xa7/0x120
May 19 11:20:41 n154-134-017 kernel: [ 672.229819][ C138] ? __pfx_do_exit+0x10/0x10
May 19 11:20:41 n154-134-017 kernel: [ 672.229824][ C138] ? _raw_spin_unlock_irq+0x2b/0x50
May 19 11:20:41 n154-134-017 kernel: [ 672.229827][ C138] ? trace_hardirqs_on+0x2e/0x1a0
May 19 11:20:41 n154-134-017 kernel: [ 672.229831][ C138] ? __kasan_check_read+0x15/0x20
May 19 11:20:41 n154-134-017 kernel: [ 672.229837][ C138] do_group_exit+0xbf/0x260
>
> > + */
> > +struct callback_head *
> > +task_work_cancel_match_all(struct task_struct *task,
> > + bool (*match)(struct callback_head *, void *data),
> > + void *data)
> > +{
> > + struct callback_head **pprev = &task->task_works;
> > + struct callback_head *work, *next;
> > + struct callback_head *head = NULL, **tail = &head;
> > + unsigned long flags;
> > +
> > + if (likely(!task_work_pending(task)))
> > + return NULL;
> > +
> > + raw_spin_lock_irqsave(&task->pi_lock, flags);
> > + work = READ_ONCE(*pprev);
> > + while (work && work != &work_exited) {
> > + next = READ_ONCE(work->next);
> > + if (!match(work, data)) {
> > + pprev = &work->next;
> > + work = next;
> > + continue;
> > + }
> > +
> > + if (!try_cmpxchg(pprev, &work, next))
> > + continue;
>
>
> IIUC, you could ignore the cmpxchg here because the following loop
> iteration on the caller would catch it and retry. In this case, it no
> retry in io_wq_cancel_tw_create, which looks weird. Did I miss something?
There is no need retry in io_wq_cancel_tw_create.
In the main teardown path, IO_WQ_BIT_EXIT has already been set by the
time io_wq_cancel_tw_create() is called.
As a result, the workqueue is already in the exit state, and normal worker
creation is no longer allowed to proceed.
However, due to a concurrency window, a small number of late create_work
items may still get queued after exit has begun.
These late arrivals are not handled by an outer retry loop; instead, they are
cleaned up by the post-add exit check in io_queue_worker_create() , which
calls io_wq_cancel_tw_create() again if needed.
Therefore, calling task_work_cancel_match_all() only once does not miss any
Work that must be canceled during the overall teardown process.
>
> > +
> > + work->next = NULL;
> > + *tail = work;
> > + tail = &work->next;
> > + work = next;
> > + }
> > + raw_spin_unlock_irqrestore(&task->pi_lock, flags);
> > +
> > + return head;
> > +}
> > +
> > static bool task_work_func_match(struct callback_head *cb, void *data)
> > {
> > return cb->func == data;
>
> --
> Gabriel Krisman Bertazi
>
© 2016 - 2026 Red Hat, Inc.