From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 67E94379EC4 for ; Fri, 31 Jul 2026 20:30:42 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529844; cv=none; b=LfcoeZ0d2QYKeByaUt8nstgHeYf4pk6rrDqX3EI8CVhQN/d3Y8F9Q1AZsa8RocZpMuL/B7JcO97Z3nixghePiFOaGgh8UE29qiC/dS8nw5Ybp6ad5obzr4tE8SV99xCHECQd0hceosOfMJc28Cqlt0x03uAqVqRajA5iKbaZHVo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529844; c=relaxed/simple; bh=k6E9T3RdSQ8la9/9Bg2w9Nv7e++ulTqGtk8Wi6e/JeA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=LxW2iV2BSwAZMzUAD29UmUyDwQa+E71L2Z/+sQUKBdroB0Q8+3V9HkozYyg+owhb0SnzvJbAAk0HNaA/VAumFzouGPJlByLZzMJ+/Hog26r1DPhlDDQaGiUWbme2LEPR4w07iW4dSricKJw4sLjrL4uKFPsDrKF2cQjZwT0exH8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bVM+wHcZ; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bVM+wHcZ" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B5F181F00ADF; Fri, 31 Jul 2026 20:30:41 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529842; bh=WiO5r9en3b3j6Z1SoF27ACEmDDEt+2SFG5r2qfaLJ+U=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=bVM+wHcZcmlCkJsJyu6Hkza2i8CtybDdk0EFmBTHq7nMdb/wFHl72S0szDjsxYLvk mq9tczjKsbzoTZMzQvz2lXBQlZh9UMJPwNgv/BGbXVM6qzJaWuw8wzzf99PAIIGNvo WBSR4b1IwZZBKEF+fd+Pmdb3CnxKIEflo/GUmN0TR9/2Sv4WQOrXajC9KEs9jgpz2W 3O0qEqNUgUZvr5DYvXudvtk1wOZVwKGJOUrxhHTWw2JhP1FkssYRIxKbnYEaDOYNHj kJzv5yL/EE7Wh6dgWwpM9dmjUrG0DskplhDboRDKq6oF4btksPYP+IkQUW64la8OpS P2uWb9okz81sA== Received: from phl-compute-02.internal (phl-compute-02.internal [10.202.2.42]) by mailfauth.phl.internal (Postfix) with ESMTP id E1A87F40077; Fri, 31 Jul 2026 16:30:40 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-02.internal (MEProxy); Fri, 31 Jul 2026 16:30:40 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdRdBIF8ZH/aPs2g1lRw7GATpB3kPZdHQ9SRq3TCfYELIaqSZraFLrXlLQvfZ+Zw L2FnPfSJd952oWr+y28mIfAsYcvh2g32U0J8iwTV49f0k9Q8hXOe/2BgVhInUEnjhmC2lX 03vYTAlN0bq9NjF03uLbRgPGL4766SVn/XbgM9h+YDdXpmZY+TYoTPsYcygLFMbtetKCiX MKKxau189OoX/tBJGac9kkLntk5lYAb6ORV5iOKO0UE2KpFdBfn5cZVhjAZINCJF+DQAjd IJX9TqJvKbTOKjLKBWundtWMSfFqKbkgkDYizysOgKGZkbbhOs1gMmPcuy0Gf54a13j8SY BsLS4AhaphsYC31TNG3SgAjWtldfFFLoqnwREbfJsO18aSConmL5I8Ko/dYwJS0xO9XdlI 08ZIoev6MdmKsrc9XJWrbgki7BUAQeMt+8cF8SkUl8lmNzbr+CVkPY7K0HZVOM4znHeF2w iBTUweF5HV6Pn6dKOoBMOaswFmdVkSFMENwkQknMeNXUVgFJ1h+9nfs0EE//kEu8ii+JsM zSsB07IV1Dp/xm1N3dazcJLt7i0+CTnQesSK37zkN8wuzdXmwDvvq7pKg2FSoxjyb4Ke6v XSZnJ4Uqh1IksIwPUQaPJf2Y8jObTUbaIEBJCz+VCzdMw+ctDsLY1swtHd4g X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:40 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Joel Fernandes Subject: [PATCH 01/24] preempt: Track NMI nesting to separate per-CPU counter Date: Fri, 31 Jul 2026 13:30:02 -0700 Message-ID: <20260731203031.13679-2-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Joel Fernandes Move NMI nesting tracking from the preempt_count bits to a separate per-CPU counter (nmi_nesting). This is to free up the NMI bits in the preempt_count, allowing those bits to be repurposed for other uses. Reduce NMI_BITS from 4 to 1, using it only to detect if we're in an NMI. The per-CPU counter currently caps nesting at 15. [boqun: Solve Steven Rostedt's comment on the BUG_ON() condition] [boqun: Use preempt_count_set() in __nmi_exit() to avoid underflow] Suggested-by: Boqun Feng Signed-off-by: Joel Fernandes Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260121223933.1568682-3-lyude@redhat.com --- include/linux/hardirq.h | 17 +++++++++++++---- include/linux/preempt.h | 9 +++++++-- kernel/softirq.c | 2 ++ tools/testing/selftests/bpf/bpf_experimental.h | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index d57cab4d4c06..8d4895531a45 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -10,6 +10,8 @@ #include #include =20 +DECLARE_PER_CPU(unsigned int, nmi_nesting); + extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); =20 @@ -102,14 +104,17 @@ void irq_exit_rcu(void); */ =20 /* - * nmi_enter() can nest up to 15 times; see NMI_BITS. + * nmi_enter() can nest - nesting is tracked in a per-CPU counter. */ #define __nmi_enter() \ do { \ lockdep_off(); \ arch_nmi_enter(); \ - BUG_ON(in_nmi() =3D=3D NMI_MASK); \ - __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ + __this_cpu_inc(nmi_nesting); \ + __preempt_count_add(HARDIRQ_OFFSET); \ + preempt_count_set(preempt_count() | NMI_MASK); \ } while (0) =20 #define nmi_enter() \ @@ -124,8 +129,12 @@ void irq_exit_rcu(void); =20 #define __nmi_exit() \ do { \ + unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + nesting =3D __this_cpu_dec_return(nmi_nesting); \ + if (!nesting) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index d964f965c8ff..586f96688325 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,6 +17,8 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) + * - bits 16-19 are the hardirq count (max # of hardirqs: 16) + * - bit 20 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -24,16 +26,19 @@ * there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * + * NMI nesting depth is tracked in a separate per-CPU variable + * (nmi_nesting) to save bits in preempt_count. + * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00f00000 + * NMI_MASK: 0x00100000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) diff --git a/kernel/softirq.c b/kernel/softirq.c index 4425d8dce44b..10af5ed859e7 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -88,6 +88,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif =20 +DEFINE_PER_CPU(unsigned int, nmi_nesting); + /* * SOFTIRQ_OFFSET usage: * diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing= /selftests/bpf/bpf_experimental.h index 67ff7882299e..e4e12001fce9 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -367,7 +367,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup,= const char *name__str, #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6EF18379C5B for ; Fri, 31 Jul 2026 20:30:44 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529847; cv=none; b=h1+DBhIR9TkldIjZktG7aZLARY5w5U66Mgsu0wDCDUpzH5s/yKeqdoMpWPEVacv34z//F+8XcJ2ZrRFZsuf9JfcjvEy4uqviMsJycYGYBVfjI40k2niddD0QgRfL7xdBPyPe9u41K45wxcyvKIUwpTTzwxNT3oDCac8FG5RwD7M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529847; c=relaxed/simple; bh=55TqpUfuAIhEFzsbKnTQx5nQGw2qkRdYWCbViSHl+0s=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=RRRETagfD2VyCi4hzuC3Lc30CfPH/r6P7DmHH7AwlmPKoYX6loJt4H7+YoYt9yvJBZ7Lyfy3tFXd3tXa0ctmi5jF8URxqCG0VQ2ZWemVZBoVTehruuhFm3O2w7LamNb4oNdSO4xMcckIT2C8+XYcPRgPSyKhcktmtV0By8q+8DQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=NgxmzQoT; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="NgxmzQoT" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3083E1F00ACF; Fri, 31 Jul 2026 20:30:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529843; bh=IgBCAm+kcVy0fow8SEhw7v0Haj3+xluBY+91XG57AmQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=NgxmzQoTwdQihVpjAZZb36FRT+W0eHXxoVe1aFqJ5wsvmfLZqOnqAfUbeiUN5aA9J R8TAGTUucWxcSCgn4PR8OfLYIbnATxybVqF1FpWcMuOuopstbfrTSaXHNGQcCE0nm4 KvAqnE4W7jQY/aVgAqTZ0f58eLWGJUhprfcVDDbch9EwFrn6A9xICBwPEgEMIo8sQt k79YiKA7q3X9+gcXxPzsOowyiE5Rjd9wlrqrkAoQyBImATqGT8vsulq0MkZHqdyQ1M Tn14/1FTVjW+56l7cDoFUVxr7kQwydFJX2ZDUfxnfo7sgTBc7EDEh+tyDX5+w28aEy eJ9GknW+R+ufg== Received: from phl-compute-03.internal (phl-compute-03.internal [10.202.2.43]) by mailfauth.phl.internal (Postfix) with ESMTP id 5D3D9F40077; Fri, 31 Jul 2026 16:30:42 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-03.internal (MEProxy); Fri, 31 Jul 2026 16:30:42 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFps7Fcl2lDQN8QuKjkQ2Twray9XFETCWlSba0KMthHKuWpzT2QuJsxYlE+Jfwxxk r4LumXM/JVSIx3esVw1aDMeAJaVHLr+c+QoTtbKWA/cjowqeMJQoXkNExL9SUShkGISMJN I4Jxy+s/E1/lkenAlXh9Q30vQuf9rA/PbrkWD4pKxMAkC+U6f3km0ithIr0S+TXfbV+4FM rMsII+vjt3nbbUrqTbYDv01BF1FGDJ2EkkIzsv92rcLPomBtS/DYv0HD/S/dv1En38CiLr u6+5RUMJmyRxjeZZwQND0Q2Cx0WP3XHFSY1h6xZNSfNwcAc/Ongt++m0x7heb8YD7Irwtn n4ZmWIflfBfwnLrvNSPJ6Jm4qPa2HoOoW5+9HICFMytbPxOT26ncADgDDkbfLhNQltB3J2 iMn0N73tbWkOyDEqOXFaJV8OaAdTDtvFLbdGjfGOsDB6X3BX9k/xrk/KDUqkIiMYVKcQF/ 8KFInmvGqihV/8tOVBGy9TS5dyCMYbpmEwoE1BoXzBxnU+RjSF7KZCSNTnADBPolzcK1rL Q3RMphBXAyhjwCHwrJoeXX7Kgz/MazHkro9i5pm8tg0J+anvYQpL8tQM01ivv77xFtE2j7 XmjgLzNkgkyS4YPpEl4pVsfzTV4m8pBHeQ5vYECPGiMT0FQEiDY9yfZZ/wCg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:41 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Boqun Feng Subject: [PATCH 02/24] preempt: Introduce HARDIRQ_DISABLE_BITS Date: Fri, 31 Jul 2026 13:30:03 -0700 Message-ID: <20260731203031.13679-3-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Boqun Feng In order to support preempt_disable()-like interrupt disabling, that is, using part of preempt_count() to track interrupt disabling nested level, change the preempt_count() layout to contain 8-bit HARDIRQ_DISABLE count. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260121223933.1568682-2-lyude@redhat.com --- include/linux/preempt.h | 16 +++++++++++----- tools/testing/selftests/bpf/bpf_experimental.h | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 586f96688325..e2d3079d3f5f 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,8 +17,9 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) - * - bits 16-19 are the hardirq count (max # of hardirqs: 16) - * - bit 20 is the NMI flag (no nesting count, tracked separately) + * - bits 16-23 are the hardirq disable count (max # of hardirq disable: 2= 56) + * - bits 24-27 are the hardirq count (max # of hardirqs: 16) + * - bit 28 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -31,29 +32,34 @@ * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 - * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00100000 + * HARDIRQ_DISABLE_MASK: 0x00ff0000 + * HARDIRQ_MASK: 0x0f000000 + * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 #define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) =20 #define __IRQ_MASK(x) ((1UL << (x))-1) =20 #define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_= DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) =20 #define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT) #define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) #define NMI_OFFSET (1UL << NMI_SHIFT) =20 diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing= /selftests/bpf/bpf_experimental.h index e4e12001fce9..0159a3d365c8 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -366,17 +366,20 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgrou= p, const char *name__str, =20 #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 #define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) =20 #define __IRQ_MASK(x) ((1UL << (x))-1) =20 #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_= DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) =20 --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4582C37A498 for ; Fri, 31 Jul 2026 20:30:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529847; cv=none; b=bO43DurbJBUmyEL549Ou41quwzCQCR54eUeQIVh4DlL4qQb6zxD0ullobC01rJLPMjCv+Km9hDWk4WiIua8Dhb7NM5dq1m6lt/ec9tvWNN7CilXWUgiD6FFvA+VOD7v3FRtPANm63eFkQjwBXE75cUZ6gK/gt+HWRz5B8rnBPYM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529847; c=relaxed/simple; bh=7WkKWIqNSdtquMO/UCT6lKa5xRPmY6l6PwpqysBoDeY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=VHzOY9xArMsmJPcP8AXv982SpviyiOtao/zdVBdw9tJSQJJPW71x438zF2GE6RYnyInRGlaF8r9M9LnaI6HzvLhmMrhcaUNdRu/m7W5JGZ8oleSbxc6oPGQooZI+djIp41PN563gqdW2DlZfrONRdtheCUbk94O3ApmjJZ0tpRg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Vss3t99o; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Vss3t99o" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B69021F00ADF; Fri, 31 Jul 2026 20:30:44 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529845; bh=WH2YlwaR5eHhEQZk5J0iMOLdzBdIQSx2iV/VlZ3YLwQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Vss3t99obNlgFO3407Gs+3ia1uSkLwvnpsCgbr3mSga8tO6dJhPh6jPLp+U2z+OIQ WluTsI9tZtW7hoCNAbNSOoyPXcvcTTk7WMcasokhm12gGiCzHfz5IXLOzmxjgA4BNl 4YcvU2y/WWEhggxt1ScLfU7B5nKruRneclK2CUzJQReKbYsA+HVWyVneBM6vpri2Pf T9t6a/zQgZSbdlvhA+Fn9rHsS7zo8cbiswEsBr5dXwuyRSwTN9wRhkAGZObW2JsugO 77vy53WvFZeC9em0vbHIZhCpycuOLxGAazsQMgCvqikKj+iahb2jmmeq1LaZ1/hpAT mITgVcvOH4XKA== Received: from phl-compute-01.internal (phl-compute-01.internal [10.202.2.41]) by mailfauth.phl.internal (Postfix) with ESMTP id E4420F40077; Fri, 31 Jul 2026 16:30:43 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-01.internal (MEProxy); Fri, 31 Jul 2026 16:30:43 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdRdBIF8ZH/aPs2g1lRw7GATpB3kPZdHQ9SRq3TCfYELIaqSZraFLrXlLQvfZ+Zw L2FnPfSJd952oWr+y28mIfAsYcvh2g32U0J8iwTV49f0k9Q8hXOe/2BgVhInUEnjhmC2lX 03vYTAlN0bq9NjF03uLbRgPGL4766SVn/XbgM9h+YDdXpmZY+TYoTPsYcygLFMbtetKCiX MKKxau189OoX/tBJGac9kkLntk5lYAb6ORV5iOKO0UE2KpFdBfn5cZVhjAZINCJF+DQAjd IJX9TqJvKbTOKjLKBWundtWMSfFqKbkgkDYizysOgKGZkbbhOs1gMmPcuy0Gf54a13j8my FlqsBMulLnIO8WoS5D1Trn6ZCGd+ly1yaYBXFWpCCCgMdnF3hWuB50Vm/U7nwcmIzEn57A VMb3xwSvkUNjJDbRYj7rplT/4tZtnlOHappT7HWGIb2+s3PS2iQU8YFyRvdTxDKie4I9xR rp9VuGabiwIaVdcw/tYPx5NFpyP5kiGXYTJEcIkM52L5RioThonEocNzM64Xhxu3CNWta7 DiFlVBUwVBZviGQzeKPKyu2J0Yhqw+jlp4KVKRW65hg+c2iNSGtYz9vCq4bm30yca0cWVS /Fym2Gi8MjeMAf5h2fF3sVJ7TsRDIthid1KFc54gk7feSUj1MhAnJnzrutUw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:43 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Boqun Feng , Heiko Carstens Subject: [PATCH 03/24] preempt: Introduce __preempt_count_{sub, add}_return() Date: Fri, 31 Jul 2026 13:30:04 -0700 Message-ID: <20260731203031.13679-4-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Boqun Feng In order to use preempt_count() to tracking the interrupt disable nesting level, __preempt_count_{add,sub}_return() are introduced, as their name suggest, these primitives return the new value of the preempt_count() after changing it. The following example shows the usage of it in local_interrupt_disable(): // increase the HARDIRQ_DISABLE bit new_count =3D __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET); // if it's the first-time increment, then disable the interrupt // at hardware level. if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D HARDIRQ_DISABLE_OFFSET) { local_irq_save(flags); raw_cpu_write(local_interrupt_disable_state.flags, flags); } Having these primitives will avoid a read of preempt_count() after changing preempt_count() on certain architectures. Acked-by: Heiko Carstens # s390 Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260121223933.1568682-4-lyude@redhat.com --- arch/arm64/include/asm/preempt.h | 18 ++++++++++++++++++ arch/s390/include/asm/preempt.h | 10 ++++++++++ arch/x86/include/asm/preempt.h | 10 ++++++++++ include/asm-generic/preempt.h | 14 ++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/pree= mpt.h index 932ea4b62042..0dd8221d1bef 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -55,6 +55,24 @@ static inline void __preempt_count_sub(int val) WRITE_ONCE(current_thread_info()->preempt.count, pc); } =20 +static inline int __preempt_count_add_return(int val) +{ + u32 pc =3D READ_ONCE(current_thread_info()->preempt.count); + pc +=3D val; + WRITE_ONCE(current_thread_info()->preempt.count, pc); + + return pc; +} + +static inline int __preempt_count_sub_return(int val) +{ + u32 pc =3D READ_ONCE(current_thread_info()->preempt.count); + pc -=3D val; + WRITE_ONCE(current_thread_info()->preempt.count, pc); + + return pc; +} + static inline bool __preempt_count_dec_and_test(void) { struct thread_info *ti =3D current_thread_info(); diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preemp= t.h index 6e5821bb047e..0a25d4648b4c 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -139,6 +139,16 @@ static __always_inline bool should_resched(int preempt= _offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) =3D=3D preempt_of= fset); } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + return val + __atomic_add(val, &get_lowcore()->preempt_count); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return __preempt_count_add_return(-val); +} + #define init_task_preempt_count(p) do { } while (0) /* Deferred to CPU bringup time */ #define init_idle_preempt_count(p, cpu) do { } while (0) diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 578441db09f0..1220656f3370 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -85,6 +85,16 @@ static __always_inline void __preempt_count_sub(int val) raw_cpu_add_4(__preempt_count, -val); } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + return raw_cpu_add_return_4(__preempt_count, val); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return raw_cpu_add_return_4(__preempt_count, -val); +} + /* * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to resch= edule * a decrement which hits zero means we have no preempt_count and should diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index 51f8f3881523..c8683c046615 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -59,6 +59,20 @@ static __always_inline void __preempt_count_sub(int val) *preempt_count_ptr() -=3D val; } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + *preempt_count_ptr() +=3D val; + + return *preempt_count_ptr(); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + *preempt_count_ptr() -=3D val; + + return *preempt_count_ptr(); +} + static __always_inline bool __preempt_count_dec_and_test(void) { /* --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9E64537BE75 for ; Fri, 31 Jul 2026 20:30:47 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529850; cv=none; b=hVIMzJcr8GkR0rFpYqfAWcHZsvX8pQsrxNzfXnWMg7vKk53IefjMfKVzCWj18Kj0ne0PWWFvOCles+t28P2z9QXlmQcspY5BsoAZXuQpYVkeC5FBbWA4hSXAI7K8EialQeSinAqyHr4UjqpUzjN6pFVBdo/csjWEvs0wN1DQJC8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529850; c=relaxed/simple; bh=YaVjrXKhfpqvIJygzREE5zKBOLGP3K/KF1IwfTEAxVo=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=VmSEJrf8dL13/kw7oqZpT2ppbnhAQksvmMtZ/mswviEIn1bTLGm597Q2U9ZRAAL9CfNArkUe6m8xaYLJtQEUcgV1D/pbKAWBM1/kJwPpIDhf0IxGmIW3rXxXZ8Qoo0TesYZLp0Q8REWAYsZdYDaSq/+U1Yj/by9vvBVwpjPIvhM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=NV1kmXLg; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="NV1kmXLg" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 488FE1F00AC4; Fri, 31 Jul 2026 20:30:46 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529846; bh=u8OFyi867RVcm393aEUm2p/F0THxOCKpMTZFIKs3ThQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=NV1kmXLgL8w4igoA7q0CvnAGyuQkefzpZQ+6Icvk4JS80KcW5OBxyQQlRW4VyH0UN 1ihm2dQyV/ibl0LaiyqZ812xFCKfTzGBEVXyb1sBDiHSzHMfUp/+BpRE+kpZp8T11W F7DmZotbQNgGOlTlpgwUaKuNH0Me9v7ITef1RsmflHjtrFa801rxP6BhDpsBRYonuF aY0LfcrbfKiEkTDNJsH3XSo0ad3PzDcCPiGph45BPaq6KOapG/G8HNUQMbXLVJLJJ+ VeiV7HEcLvxXx8Ogl2FVRpj1T9o7nipjwHKTXJnhnRpJzbhCcepGPv2n4pMdRY5+SQ SJooiGxTIOkCQ== Received: from phl-compute-02.internal (phl-compute-02.internal [10.202.2.42]) by mailfauth.phl.internal (Postfix) with ESMTP id 77711F40077; Fri, 31 Jul 2026 16:30:45 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-02.internal (MEProxy); Fri, 31 Jul 2026 16:30:45 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFYc6116FG2quGqEuyKIxKnFMEmlYb32kQQ7CtbFGYD9qEYkru11crvm5k1myTjrD mprk+Q3zryg6Sw+VBlHk+Fh2dsP+vkQ+R6SId6D6PQHJ2MNR0r6gfBCLOd3kp8148kZ7F/ KLgIA+5OGrlMzKnGLja63bpFSrAh19/D1KowErF7/fMLoXr6/uBxuTdAfInIovWJuydywf zjiEDrBy2ZQGTG+wDuq3UDJInF4v///JGqWliNx3oB8ndvgTYjGjYM24OlzqaeVHYGQ/UH n/uNgL2CgUXmcpbS+mAJlGjkfOgTDC/OpURLEQyF+W0WAZBtt5IfiFk+NFnrr0x50+KqCH wBaR2L1NGEhuJ5gFtE9UfELjCUn20ByVkTai443w5pGYQyilOMUTcSvnCN5pJbLS/Xl8Km /2t5w9SqtaJ6T5RX/6w/8CArOQTwePpOIBreezPuQw2Zuwr3mEfNdDsNNde15wJtoO3T5y U0DQYel3bjf/sEaGpRn1PWSr9xiCPwyDv1i14QGnQZp/d52dEOyAuxUT1obdfvm/pK2h26 7SKHwU14cj5U8T7XgUXNRb9GMlLV48lYmE/KJJ/7VNaGvRbun958yK+0nzFe5CFkaMjtW/ Ey233q2kGhKJlmOXFWY3z3YsUtPOKR/ZviUo5Eheuojtsq2291zhyyTVU0UQ X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:44 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Stafford Horne Subject: [PATCH 04/24] openrisc: Include in smp.h Date: Fri, 31 Jul 2026 13:30:05 -0700 Message-ID: <20260731203031.13679-5-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul While OpenRISC currently doesn't fail to build upstream, it appears that include in the right headers is enough to break that - primarily because OpenRISC's asm/smp.h header doesn't actually provide any definition for struct cpumask. Which means the only reason we aren't failing to build kernel is because we've been lucky enough that every spot including asm/smp.h already has definitions for struct cpumask pulled in. This became evident when trying to work on a patch series for adding ref-counted interrupt enable/disables to the kernel, where introducing a new interrupt_rc.h header suddenly introduced a build error on OpenRISC: In file included from include/linux/interrupt_rc.h:17, from include/linux/spinlock.h:60, from include/linux/mmzone.h:8, from include/linux/gfp.h:7, from include/linux/mm.h:7, from arch/openrisc/include/asm/pgalloc.h:20, from arch/openrisc/include/asm/io.h:18, from include/linux/io.h:12, from drivers/irqchip/irq-ompic.c:61: arch/openrisc/include/asm/smp.h:21:59: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration 21 | extern void arch_send_call_function_ipi_mask(const struct cpum= ask *mask); | ^~~~= ~~~ arch/openrisc/include/asm/smp.h:23:54: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration 23 | extern void set_smp_cross_call(void (*)(const struct cpumask *= , unsigned int)); | ^~~~~~~ drivers/irqchip/irq-ompic.c: In function 'ompic_of_init': >> drivers/irqchip/irq-ompic.c:191:28: error: passing argument 1 of 'set_smp_cross_call' from incompatible pointer type [-Werror=3Dincompatible-pointer-types] 191 | set_smp_cross_call(ompic_raise_softirq); | ^~~~~~~~~~~~~~~~~~~ | | | void (*)(const struct cpumask *, un= signed int) arch/openrisc/include/asm/smp.h:23:32: note: expected 'void (*)(const struct cpumask *, unsigned int)' but argument is of type 'void (*)(const struct cpumask *, unsigned int)' 23 | extern void set_smp_cross_call(void (*)(const struct cpumask *= , unsigned int)); To fix this, let's take an example from the smp.h headers of other architectures (x86, hexagon, arm64, probably more): just include linux/cpumask.h at the top. Signed-off-by: Lyude Paul Acked-by: Stafford Horne Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260121223933.1568682-5-lyude@redhat.com --- arch/openrisc/include/asm/smp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/openrisc/include/asm/smp.h b/arch/openrisc/include/asm/sm= p.h index 007296f160ef..84653aaffa96 100644 --- a/arch/openrisc/include/asm/smp.h +++ b/arch/openrisc/include/asm/smp.h @@ -9,6 +9,8 @@ #ifndef __ASM_OPENRISC_SMP_H #define __ASM_OPENRISC_SMP_H =20 +#include + #include #include =20 --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8BA99378D9C for ; Fri, 31 Jul 2026 20:30:48 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529852; cv=none; b=ZMATfP5Fn9mEkjQyCDQ4oirknlepOUa6nDLQhyMYI+dHpkcqcLrGgZAW8Ejz37GHv/CpHxUh+HUynCpP0eePc5h8EPMNzWqVNoT8xmqEC0wkupsYlnE2a9F5L9j/78mRU8bKqkNmUkmdAgtyUh0JDj5oeaQaeT02syu43uI7y+4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529852; c=relaxed/simple; bh=to0RP4Su+MQgZQGe+JSAj9EmfBFi1aiWWRZPL1K6doM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=mS/n3Ay2c8cAH720WF3KPeMYmjGOmMWjBHAt41UlG7WPjvyrIsjz/OCAguQtpblmGRHyMiJjj0bAzz+fSg8f9WyN+d2XEVlt56ELvrrse2Xj0XPUXq3EirbMWg77+pqizA0Fh83PeHOQBeqIeOlE/Aucp4DfuwSaZbuMpM12n5s= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=RoQceZ+h; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="RoQceZ+h" Received: by smtp.kernel.org (Postfix) with ESMTPSA id DA29D1F00ACF; Fri, 31 Jul 2026 20:30:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529848; bh=zaL3yYnLyWfmGyr/NqPQXNi1BCOD+TsBumBelPwYXZQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=RoQceZ+hUp1rg8GPyHAGDnRvHPFS6xqsRB+t0wUW1Ik7cBr+WCoElE0OICEUPmWUU jsr9wgJS3vvAtEiByxvWpNkrLRtZugnF6oBIlORt6UbxBPWZde2jkstaCGC1bTKXLk CviQkiLoCvACnZsfsFGQvgA7sUWlNThmCzPsATMW+mucQMWcGiGmztGcdO7yFFIn+P 7WiVstZSLAjEyDYUBa0yZ907emNphUyLqGONb6mh8XVvKC+oB1uXpRRCJo3Uk8Ezdd c/lz3/1urYU3Ti5ga/tKZBfjw8U9y/kXYQzHZdsH1As2r3cyNGq6Oh6yieUArcqDjf w1DBNuFXSXwpA== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 1483AF40077; Fri, 31 Jul 2026 16:30:47 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-05.internal (MEProxy); Fri, 31 Jul 2026 16:30:47 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdRdBIF8ZH/aPs2g1lRw7GATpB3kPZdHQ9SRq3TCfYELIaqSZraFLrXlLQvfZ+Zw L2FnPfSJd952oWr+y28mIfAsYcvh2g32U0J8iwTV49f0k9Q8hXOe/2BgVhInUEnjhmC2lX 03vYTAlN0bq9NjF03uLbRgPGL4766SVn/XbgM9h+YDdXpmZY+TYoTPsYcygLFMbtetKCiX MKKxau189OoX/tBJGac9kkLntk5lYAb6ORV5iOKO0UE2KpFdBfn5cZVhjAZINCJF+DQAjd IJX9TqJvKbTOKjLKBWundtWMSfFqKbkgkDYizysOgKGZkbbhOs1gMmPcuy0Gf54a13j8Sm UmcOx+4EJMNfu9gA3gGgYz1gzu+a4+d94aVOKt5anODL+/3E7yxXwwzjNpfdo40eps/Wyf gPSt3mJoNlaSArxLVpobfg9Ft9lBRSYviLs5I+hPo0Nh5dK3m2O5D1pszFbqkNE49y/8zQ 1oK7tEwgDcnpBynAAhBEWfuERsmQvR6/YJm4hUhTGuweThq921ru5hhpb95zIadA+T9qd+ m4HEsRWEVhYgp+JiNJ7Yjmj7u9fLG3RBAtst1BOcQHlog1qYg36w5hLwCF1/KK2S7xGZoV qgfDgcC7mkgta4Y+gZYU/26LFoYzoIb08s0aiqp1PLwkH/NpqhRq6nLiC2lw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:46 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Boqun Feng Subject: [PATCH 05/24] irq & spin_lock: Add counted interrupt disabling/enabling Date: Fri, 31 Jul 2026 13:30:06 -0700 Message-ID: <20260731203031.13679-6-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Boqun Feng Currently the nested interrupt disabling and enabling is present by _irqsave() and _irqrestore() APIs, which are relatively unsafe, for example: spin_lock_irqsave(l1, flag1); spin_lock_irqsave(l2, flag2); spin_unlock_irqrestore(l1, flags1); // accesses to interrupt-disable protect data will cause races. This is even easier to triggered with guard facilities: unsigned long flag2; scoped_guard(spin_lock_irqsave, l1) { spin_lock_irqsave(l2, flag2); } // l2 locked but interrupts are enabled. spin_unlock_irqrestore(l2, flag2); (Hand-to-hand locking critical sections are not uncommon for a fine-grained lock design) And because this unsafety, Rust cannot easily wrap the interrupt-disabling locks in a safe API, which complicates the design. To resolve this, introduce a new set of interrupt disabling APIs: * local_interrupt_disable(); * local_interrupt_enable(); They work like local_irq_save() and local_irq_restore() except that 1) the outermost local_interrupt_disable() call save the interrupt state into a percpu variable, so that the outermost local_interrupt_enable() can restore the state, and 2) a percpu counter is added to record the nest level of these calls, so that interrupts are not accidentally enabled inside the outermost critical section. Also add the corresponding spin_lock primitives: spin_lock_irq_disable() and spin_unlock_irq_enable(), as a result, code as follow: spin_lock_irq_disable(l1); spin_lock_irq_disable(l2); spin_unlock_irq_enable(l1); // Interrupts are still disabled. spin_unlock_irq_enable(l2); doesn't have the issue that interrupts are accidentally enabled. This also makes the wrapper of interrupt-disabling locks on Rust easier to design. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260121223933.1568682-6-lyude@redhat.com --- include/linux/interrupt_rc.h | 67 ++++++++++++++++++++++++++++++++ include/linux/preempt.h | 4 ++ include/linux/spinlock.h | 23 +++++++++++ include/linux/spinlock_api_smp.h | 41 +++++++++++++++++++ include/linux/spinlock_api_up.h | 16 ++++++++ include/linux/spinlock_rt.h | 18 +++++++++ kernel/locking/spinlock.c | 29 ++++++++++++++ kernel/softirq.c | 14 ++++++- 8 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 include/linux/interrupt_rc.h diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h new file mode 100644 index 000000000000..dd4444c61330 --- /dev/null +++ b/include/linux/interrupt_rc.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/linux/interrupt_rc.h - refcounted local processor interrupt + * management. + * + * Since the implementation of this API currently depends on + * local_irq_save()/local_irq_restore(), we split this into it's own heade= r to + * make it easier to include without hitting circular header dependencies. + */ + +#ifndef __LINUX_INTERRUPT_RC_H +#define __LINUX_INTERRUPT_RC_H + +#include +#include +#ifdef CONFIG_SMP +#include +#endif + +/* Per-cpu interrupt disabling state for local_interrupt_{disable,enable}(= ) */ +struct interrupt_disable_state { + unsigned long flags; +}; + +DECLARE_PER_CPU(struct interrupt_disable_state, local_interrupt_disable_st= ate); + +static inline void local_interrupt_disable(void) +{ + unsigned long flags; + int new_count; + + WARN_ON_ONCE(in_nmi()); + + new_count =3D hardirq_disable_enter(); + + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ + + if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D HARDIRQ_DISABLE_OFFSET) { + local_irq_save(flags); + raw_cpu_write(local_interrupt_disable_state.flags, flags); + } +} + +static inline void local_interrupt_enable(void) +{ + int new_count; + + new_count =3D hardirq_disable_exit(); + + if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D 0) { + unsigned long flags; + + flags =3D raw_cpu_read(local_interrupt_disable_state.flags); + local_irq_restore(flags); + /* + * TODO: re-read preempt count can be avoided, but it needs + * should_resched() taking another parameter as the current + * preempt count + */ +#ifdef CONFIG_PREEMPTION + if (should_resched(0)) + __preempt_schedule(); +#endif + } +} + +#endif /* !__LINUX_INTERRUPT_RC_H */ diff --git a/include/linux/preempt.h b/include/linux/preempt.h index e2d3079d3f5f..33fc4c814a9f 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context= _level(void) #define in_softirq() (softirq_count()) #define in_interrupt() (irq_count()) =20 +#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) = >> HARDIRQ_DISABLE_SHIFT) +#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE= _OFFSET) +#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_= OFFSET) + /* * The preempt_count offset after preempt_disable(); */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 241277cd34cf..3d405cc4c121 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -57,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *= lock) __releases(lock) #endif =20 #define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock) #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock) #define raw_spin_unlock(lock) _raw_spin_unlock(lock) #define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock) +#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock) =20 #define raw_spin_unlock_irqrestore(lock, flags) \ do { \ @@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *l= ock) __releases(lock) =20 #define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lo= ck, &(flags)) =20 +#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(l= ock) + #ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ #include @@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *= lock) raw_spin_lock_irq(&lock->rlock); } =20 +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + raw_spin_lock_irq_disable(&lock->rlock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ raw_spin_lock_irqsave(spinlock_check(lock), flags); \ @@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t= *lock) raw_spin_unlock_irq(&lock->rlock); } =20 +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) __no_context_analysis +{ + raw_spin_unlock_irq_enable(&lock->rlock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsig= ned long flags) __releases(lock) __no_context_analysis { @@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spin= lock_t *lock, unsigned lon } #define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(fl= ags)) =20 +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) __no_context_analysis +{ + return raw_spin_trylock_irq_disable(&lock->rlock); +} + /** * spin_is_locked() - Check whether a spinlock is locked. * @lock: Pointer to the spinlock. diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_= smp.h index bda5e7a390cd..07a94ba1d760 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct loc= kdep_map *map) void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock); void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) __acquires(lock); +void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock); =20 unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) __acquires(lock); @@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock)= __cond_acquires(true, void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock= ); +void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releas= es(lock); void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) __releases(lock); @@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsign= ed long flags) #define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock) #endif =20 +/* Use the same config as spin_lock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ +#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE #define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock) #endif @@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsign= ed long flags) #define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock) #endif =20 +/* Use the same config as spin_unlock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ +#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(loc= k) +#endif + #ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE #define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrest= ore(lock, flags) #endif @@ -105,6 +118,18 @@ static __always_inline bool _raw_spin_trylock_irq(raw_= spinlock_t *lock) return false; } =20 +static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *= lock) + __cond_acquires(true, lock) +{ + local_interrupt_disable(); + if (do_raw_spin_trylock(lock)) { + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); + return true; + } + local_interrupt_enable(); + return false; +} + static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock= , unsigned long *flags) __cond_acquires(true, lock) { @@ -143,6 +168,14 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t = *lock) LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); } =20 +static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + local_interrupt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); + LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); +} + static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock) __no_context_analysis { @@ -188,6 +221,14 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_= t *lock) preempt_enable(); } =20 +static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock) + __releases(lock) +{ + spin_release(&lock->dep_map, _RET_IP_); + do_raw_spin_unlock(lock); + local_interrupt_enable(); +} + static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock) { diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_u= p.h index a9d5c7c66e03..e4de8bb26a15 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -42,6 +42,9 @@ #define __LOCK_IRQSAVE(lock, flags, ...) \ do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0) =20 +#define __LOCK_IRQ_DISABLE(lock, ...) \ + do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0) + #define ___UNLOCK_(lock) \ do { __release(lock); (void)(lock); } while (0) =20 @@ -61,6 +64,10 @@ #define __UNLOCK_IRQRESTORE(lock, flags, ...) \ do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0) =20 +#define __UNLOCK_IRQ_ENABLE(lock, ...) \ + do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0) + + #define _raw_spin_lock(lock) __LOCK(lock) #define _raw_spin_lock_nested(lock, subclass) __LOCK(lock) #define _raw_read_lock(lock) __LOCK(lock, shared) @@ -70,6 +77,7 @@ #define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared) #define _raw_write_lock_bh(lock) __LOCK_BH(lock) #define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock) +#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock) #define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared) #define _raw_write_lock_irq(lock) __LOCK_IRQ(lock) #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) @@ -97,6 +105,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_sp= inlock_t *lock) return 1; } =20 +static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *l= ock) + __cond_acquires(true, lock) +{ + __LOCK_IRQ_DISABLE(lock); + return 1; +} + static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock,= unsigned long *flags) __cond_acquires(true, lock) { @@ -132,6 +147,7 @@ static __always_inline int _raw_write_trylock_irqsave(r= wlock_t *lock, unsigned l #define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock) #define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared) #define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock) +#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock) #define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared) #define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock) #define _raw_spin_unlock_irqrestore(lock, flags) \ diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 373618a4243c..560d06384e0c 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lo= ck) rt_spin_lock(lock); } =20 +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) +{ + rt_spin_lock(lock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ @@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t= *lock) rt_spin_unlock(lock); } =20 +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) +{ + rt_spin_unlock(lock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) @@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spi= nlock_t *lock, =20 #define spin_trylock(lock) rt_spin_trylock(lock) =20 +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) +{ + return rt_spin_trylock(lock); +} + #define spin_trylock_bh(lock) rt_spin_trylock_bh(lock) =20 #define spin_trylock_irq(lock) rt_spin_trylock(lock) diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index b42d293da38b..764641f6ec57 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -129,6 +129,19 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##= _t *lock) \ */ BUILD_LOCK_OPS(spin, raw_spinlock, __acquires); =20 +/* No rwlock_t variants for now, so just build this function by hand */ +static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + for (;;) { + local_interrupt_disable(); + if (likely(do_raw_spin_trylock(lock))) + break; + local_interrupt_enable(); + + arch_spin_relax(&lock->raw_lock); + } +} + #ifndef CONFIG_PREEMPT_RT BUILD_LOCK_OPS(read, rwlock, __acquires_shared); BUILD_LOCK_OPS(write, rwlock, __acquires); @@ -176,6 +189,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlo= ck_t *lock) EXPORT_SYMBOL(_raw_spin_lock_irq); #endif =20 +#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ +noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + __raw_spin_lock_irq_disable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable); +#endif + #ifndef CONFIG_INLINE_SPIN_LOCK_BH noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) { @@ -208,6 +229,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spin= lock_t *lock) EXPORT_SYMBOL(_raw_spin_unlock_irq); #endif =20 +#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ +noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) +{ + __raw_spin_unlock_irq_enable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable); +#endif + #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) { diff --git a/kernel/softirq.c b/kernel/softirq.c index 10af5ed859e7..d1ab1799794c 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -88,6 +88,9 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif =20 +DEFINE_PER_CPU(struct interrupt_disable_state, local_interrupt_disable_sta= te); +EXPORT_PER_CPU_SYMBOL_GPL(local_interrupt_disable_state); + DEFINE_PER_CPU(unsigned int, nmi_nesting); =20 /* @@ -728,7 +731,16 @@ static inline void __irq_exit_rcu(void) #endif account_hardirq_exit(current); preempt_count_sub(HARDIRQ_OFFSET); - if (!in_interrupt() && local_softirq_pending()) { + /* + * Interrupts may happen between hardirq_disable_enter() and + * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes + * softirq here, we may have a softirq handler calling + * local_interrupt_disable() but it won't disable the irq because + * hardirq disabling count is already 1, hence we need to prevent + * invoking softirq when a local_interrupt_disable() is ongoing. + */ + if (!in_interrupt() && !hardirq_disable_count() && + local_softirq_pending()) { /* * If we left hrtimers unarmed, make sure to arm them now, * before enabling interrupts to run SoftIRQ. --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CB7AA37CD41; Fri, 31 Jul 2026 20:30:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529854; cv=none; b=J2+6nvDtLN2pUXnhWXxknJBIXIcezMTjQ34eD2PdyqJSg90g7jXbHL+JahKg8X75MHWw1kpvqjfmCLhIG4rNoWYEZS9uaWWRGS5szoLtDDDqYibghLfkFF2ysbH8sFx96u1QbMSRprbOJn46i1So6TNKSkGSTlNMEzm5t2IEwTg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529854; c=relaxed/simple; bh=bGbp7ZAovMFSYOk12em7HB3rK+s0+f5BMcq+64sn+xI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gVmCXvep9dbvr0ldzwNLW0fFEFni7a+xUCK9OpDKavkgsVX8abrogj4EpJvEoPbgGfUp8ruk4ywY+KgRS/6o+rfzeI0YgN/RfQKzRIcsMGI87wcQy9F58PdrxqrSz0df/btsJmQNA+JHjHzvsqHffdO1MSDUCVDuPEU7c4odh24= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=N18D/Gi/; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="N18D/Gi/" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 437231F01558; Fri, 31 Jul 2026 20:30:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529849; bh=5OPx7quzXUSGt6ByZC5s4NaWSZlJv3l+K0FTPDFDBPE=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=N18D/Gi/z4RcA+Ro9K7tMf1Mg9WDD670V/x6FLoA7JndmUFsQa9QVnFsXT2e00Wgw jaz+nAg57XF35M4zVSh8kM/rUm7eIDw3nnrIoDjf7rrrurjKKO58lNbBL0+mC/G0pq h9+Bt4l1LREMs5Jp+30VFpUN0b2Kbu97TJuS8JclmrOrgCs3FADgiwVaU+j2EBdxUi /WJLe84/SeYFgG+M2YlNSqxZkaqnPVHoKM+7LzeoV4ioCzjjdlbKhpVBNaEhE5r7tt JF2YKNkyPhaGqxLB0CZ0bJsdJ9D3FImiKxmkEMcoAOW/7KKPK151X27x9d1MJcbpgZ KjnmvBh+9AmFg== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 71BD3F4007A; Fri, 31 Jul 2026 16:30:48 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-05.internal (MEProxy); Fri, 31 Jul 2026 16:30:48 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdRdBIF8ZH/aPs2g1lRw7GATpB3kPZdHQ9SRq3TCfYELIaqSZraFLrXlLQvfZ+Zw L2FnPfSJd952oWr+y28mIfAsYcvh2g32U0J8iwTV49f0k9Q8hXOe/2BgVhInUEnjhmC2lX 03vYTAlN0bq9NjF03uLbRgPGL4766SVn/XbgM9h+YDdXpmZY+TYoTPsYcygLFMbtetKCiX MKKxau189OoX/tBJGac9kkLntk5lYAb6ORV5iOKO0UE2KpFdBfn5cZVhjAZINCJF+DQAjd IJX9TqJvKbTOKjLKBWundtWMSfFqKbkgkDYizysOgKGZkbbhOs1gMmPcuy0Gf54a13j8cR pRoCl8cr0A8gEDTfqmAcSL6gX7sjXZrmC7yrKNjSu0IamfpnIG/9xd76hqh+6hv2ROKCsD swFhlt6jYTOmO5PBxXpLgRjGtqh360dZxELLjD2t4JwnvE4QfrX0eD+EODi97V2rgMQros w0a+jIWhVLYesU1viaqj28r8Tml6H150tNPJMql9EtjA/9lQKhPGKQj3wj4oizPjHPeQw5 gG0x3cF9AUb2rVeEPZnpg31lekXFlpABqMhF/36zkGwg47FDc6/7SwSRa2HZZ3OwFELKbk SbOI22rZQ3ZUArwpqOq0nUZn+tIIRH7CM5TEPtzGjPGJZYtJ6zIzeRABtblQ X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:47 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH 06/24] irq: Add KUnit test for refcounted interrupt enable/disable Date: Fri, 31 Jul 2026 13:30:07 -0700 Message-ID: <20260731203031.13679-7-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul While making changes to the refcounted interrupt patch series, at some point on my local branch I broke something and ended up writing some kunit tests for testing refcounted interrupts as a result. So, let's include these tests now that we have refcounted interrupts. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260121223933.1568682-7-lyude@redhat.com --- kernel/irq/Makefile | 1 + kernel/irq/refcount_interrupt_test.c | 109 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 kernel/irq/refcount_interrupt_test.c diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 86a2e5ae08f9..44c4d6fc502a 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) +=3D affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) +=3D debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) +=3D matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) +=3D irq_test.o +obj-$(CONFIG_KUNIT) +=3D refcount_interrupt_test.o diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_int= errupt_test.c new file mode 100644 index 000000000000..b4f224595f26 --- /dev/null +++ b/kernel/irq/refcount_interrupt_test.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for refcounted interrupt enable/disables. + */ + +#include +#include + +#define TEST_IRQ_ON() KUNIT_EXPECT_FALSE(test, irqs_disabled()) +#define TEST_IRQ_OFF() KUNIT_EXPECT_TRUE(test, irqs_disabled()) + +/* =3D=3D=3D=3D=3D Test cases =3D=3D=3D=3D=3D */ +static void test_single_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); +} + +static void test_nested_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_multiple_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_irq_save(struct kunit *test) +{ + unsigned long flags; + + local_irq_save(flags); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_irq_save(flags); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static struct kunit_case test_cases[] =3D { + KUNIT_CASE(test_single_irq_change), + KUNIT_CASE(test_nested_irq_change), + KUNIT_CASE(test_multiple_irq_change), + KUNIT_CASE(test_irq_save), + {}, +}; + +/* (init and exit are the same */ +static int test_init(struct kunit *test) +{ + TEST_IRQ_ON(); + + return 0; +} + +static void test_exit(struct kunit *test) +{ + TEST_IRQ_ON(); +} + +static struct kunit_suite refcount_interrupt_test_suite =3D { + .name =3D "refcount_interrupt", + .test_cases =3D test_cases, + .init =3D test_init, + .exit =3D test_exit, +}; + +kunit_test_suite(refcount_interrupt_test_suite); +MODULE_AUTHOR("Lyude Paul "); +MODULE_DESCRIPTION("Refcounted interrupt unit test suite"); +MODULE_LICENSE("GPL"); --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9360237C90E for ; Fri, 31 Jul 2026 20:30:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529854; cv=none; b=H7N4cVxIz/RfABSA9xbf+K1/LHvnnC8ocFgYRS37QA0HJMaNUtoM2yumNbYd5u4Re4dAAWbzXxSmvlfYcYz4SuM8SyitdJ0KNKZT5AMiY3jV99uuo3odJSKkwABPYnhZsvgXiPDKQcMtVEilhxTnqqimLp/K2JiIFss87K4fuIA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529854; c=relaxed/simple; bh=FgYJuF5UfyUoA8c8MpgDHb5r92lhVu0ntxZrOdq6QEA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=RW5o43Pn+OCVO1S1cQJyANyQAUzlW669OVurUCP7RNWo5NsLIKXnSjhP0+LZF8+h//dBi6900PopatZYGsbmTak+SgFBIVU3176YnlYFtJSn4EuJsEYXKYfwen+ERgxeQicYaQ9SkatPwSWaMYQ5vutBiyaIPvV2KReDKMO8IHs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=SaQ1EgoO; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="SaQ1EgoO" Received: by smtp.kernel.org (Postfix) with ESMTPSA id AEFB71F00ACA; Fri, 31 Jul 2026 20:30:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529851; bh=uhAZ4b5JiVFMWBV6D1eFcbA6MpCp1fwtNKBgvlUsvfU=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=SaQ1EgoO6FQhYP552B3rYgJv+HT2nutzKj5j5OsX2A+fDgScuRF7VnoxK4UdTltmV ZmdLZottVvy7sbDpi5WEPKVAh5/wZ5By/H2eF1vum6jX1o4jozWAjQf7yrXx6a1IQN UqG0jrWz7v63o+f4yY7WhZ5bEaa9olsuz5HaETssCTRrTK9f5Rsg7GF3iOBR3xzQiR ulV3S4dq0nHbUwCmjKMvgWHyeQfyPJkOqiOblsPQC9bHc7Pz1vITXnMDWrkDkF0aSB qGkraG5H9gBDxbAS+vxGVkBQ3jPgHSy3mnxl2aZpTHnGxG6LtTb6MnEK3gLQhrtj8g WpF3SNUCAe3+g== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id DD43CF40077; Fri, 31 Jul 2026 16:30:49 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-06.internal (MEProxy); Fri, 31 Jul 2026 16:30:49 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdRdBIF8ZH/aPs2g1lRw7GATpB3kPZdHQ9SRq3TCfYELIaqSZraFLrXlLQvfZ+Zw L2FnPfSJd952oWr+y28mIfAsYcvh2g32U0J8iwTV49f0k9Q8hXOe/2BgVhInUEnjhmC2lX 03vYTAlN0bq9NjF03uLbRgPGL4766SVn/XbgM9h+YDdXpmZY+TYoTPsYcygLFMbtetKCiX MKKxau189OoX/tBJGac9kkLntk5lYAb6ORV5iOKO0UE2KpFdBfn5cZVhjAZINCJF+DQAjd IJX9TqJvKbTOKjLKBWundtWMSfFqKbkgkDYizysOgKGZkbbhOs1gMmPcuy0Gf54a13j8O7 Pk0CaWNF4G2R5ZoK4bk8SME6AmzzoXAliTUBn962QTJljkRfeH42W/7Y8fccbcnR66n6HF y8A5vD9zerb9AXB/Q929RxYZHvVVSM2335g+gQCj665Cv/WLYIg6MsWj6qtRLeZjifW+Pv wfdFbJOXH2qYz1tJuFJm5RAmacbjKM1Xrf1fFSvkf+I5NoGuRSQc8mXDo7W9Xjw2bvUhOG 2wlo9TX659fnzPoabF4MfwGUGNskrgd8FVpxQK194AoCtKBnFLkeiyGpWbwSuC3x/kTF1Z 4pWS7Rvf6NiFH7OlcMvbzcIYfucI8mpyG+kYMRZT1zkTt9iMa+WhDsXoyURQ X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:49 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Boqun Feng Subject: [PATCH 07/24] locking: Switch to _irq_{disable,enable}() variants in cleanup guards Date: Fri, 31 Jul 2026 13:30:08 -0700 Message-ID: <20260731203031.13679-8-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Boqun Feng The semantics of various irq disabling guards match what *_irq_{disable,enable}() provide, i.e. the interrupt disabling is properly nested, therefore it's OK to switch to use *_irq_{disable,enable}() primitives. Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260121223933.1568682-17-lyude@redhat.com --- include/linux/spinlock.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 3d405cc4c121..a9d169dad6d4 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -572,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acq= uires(_T), __releases(*(raw #define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(= raw_spinlock_nested, _T) =20 DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t, - raw_spin_lock_irq(_T->lock), - raw_spin_unlock_irq(_T->lock)) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(= raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw= _spinlock_irq, _T) =20 -DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->= lock)) +DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disa= ble(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __release= s(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS= (raw_spinlock_irq_try, _T) =20 @@ -618,13 +618,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_= T), __releases(*(spinlock_t #define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinloc= k_try, _T) =20 DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t, - spin_lock_irq(_T->lock), - spin_unlock_irq(_T->lock)) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spin= lock_t **)_T)) #define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinloc= k_irq, _T) =20 DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try, - spin_trylock_irq(_T->lock)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(= spinlock_t **)_T)) #define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spi= nlock_irq_try, _T) =20 --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AE35A24A067 for ; Fri, 31 Jul 2026 20:30:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529855; cv=none; b=UDdTi9XDijQQqcoldp2tcu8SB/zbgT23jpOfqr1IHFRzrc7tzfcgnzhfkRFJYrfNyTsnDPnqCuYZg4C7QgMif7iF8iRjXjDsS52W4fuUktflESCxs36baLJ3twb+Cm5Rm64CbOxl7cGE3uxtLandZ+qXqPFbN6Q+lONBxRYNDeg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529855; c=relaxed/simple; bh=8OKC93kjB3OAypdTgsLvav/Sby5CFIUSiI4GpwugfeM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ZV9u2sMoMSCUz2+aKz2vWLz4awuRBdA+/OqInqNC2Gpqyu5NiIfteWzurNnx33xOe1z6na7WhaXPBe6zPVWHoOhZ9JWoUyvNzdIzfQ5uX+Qe9wgNbxH4VXE9+OYZc1xU7AtRGPfMqGRvdU8/oqv2EMkCG72OL4Hx3+QWpwt65IM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=htBRWhnk; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="htBRWhnk" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1AE9F1F01559; Fri, 31 Jul 2026 20:30:52 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529852; bh=IbUwG4eVfO2Hr5V4NtBdJhOCd9mCcXdlK71q0O4GZ2w=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=htBRWhnkYkIibQ3i/CWMFx8xJlmExYUtHIORxKGTHNEHzPQsU54XIlEvyI0iRJzC9 5EWxTNpSFcAXH9ds9/UwbNmQU+4rW1UAOcNLnV/5LBLK2Vr4rqb097sBHu9UsNXx6M Rvs9DM1piPn2asSNYcvxlKnkzncJZ1m4XllvpzyvpZyadrUVpNj+e01I1bQ1ndp4C8 KwCp8iHmDIUxAZ/LrJ6yZewtiIzEeMFEm425ZZSOTvk4F5Dyr6bA/k4NespxD+vaDu t3CQvRXFKTkBaihkh2oPiwbY98ChPeHKgkmEtwDGkcol4hp87ttQ9I/wUjr28Kyub5 +41C9MFv/q/0Q== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id 49D0EF4007A; Fri, 31 Jul 2026 16:30:51 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-06.internal (MEProxy); Fri, 31 Jul 2026 16:30:51 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFIC2VVTLK13uDkALWshPgkkwS9DC5Q8ljHaA42IPRnmgkrxQSeK5HYsDRhN4GhLk 6Lg9dLbHgT0DH+qVaNoW+UUFxptAXHUIdqGwEJPczm0ywmTmF9/CSNIQrGZfaA0/SYDXt9 FCHeSX4ZePBeVl3Top6EjJd42yXsFqp+L0LPuLnabUXIo8fT3yX5hMg5WmOramFVWkXrXb dtjJi0qTpjt7TXVawGUwOyoJ/Q6jP1wNaVQH+6xYGS7UXVpHVglHDWcX7SjmpbiByeHS7C oLMM3JHkGXdP7WkAt31sGvl3IAmYfCODupTHvovIO6+oDBZmEBw2YefLw5AUKyninHXIXr uU/aqZFDbwNRTuq+1lI9CTvFVdIPMn2W/4VbpaHwADVTXn7+LRw/juI6GV1qY58R3RNgFQ W3lq0jHm9OqmqYu76jnoEZYpMtIHUw7gsMYB0j6tSirJ8Gb9xN6hizDFmhtK5jFBWtNM7e 3v34M0evk1u2PrF+7KdfkDWe0s+5yP56Q5k9D1bqwESGSlfvHh7PtiJmNDhrFsFIcBJlqz 9+QlAOSbV+rp5rFAlBW8OOYAr/K0c++465XjATKopuI70y7VP6Z1UQnse93nlYdQefsPME JX3rVMdSY391nQrwT0v34tVUP7notG4f4YvRMj07GI1c7YQ0nKuNV6XLTD0Q X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:50 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH 08/24] sched: Remove the unused preempt_offset parameter of __cant_sleep() Date: Fri, 31 Jul 2026 13:30:09 -0700 Message-ID: <20260731203031.13679-9-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The preempt_offset is always 0 in all the callsites of __cant_sleep(), hence remove it. It also allows us to clear the code a bit by stopping using a "preempt_count() > .." comparison. Signed-off-by: Boqun Feng --- include/linux/kernel.h | 4 ++-- kernel/sched/core.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index e5570a16cbb1..24414c79e59a 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -72,7 +72,7 @@ extern int dynamic_might_resched(void); #ifdef CONFIG_DEBUG_ATOMIC_SLEEP extern void __might_resched(const char *file, int line, unsigned int offse= ts); extern void __might_sleep(const char *file, int line); -extern void __cant_sleep(const char *file, int line, int preempt_offset); +extern void __cant_sleep(const char *file, int line); extern void __cant_migrate(const char *file, int line); =20 /** @@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line); * this macro will print a stack trace if it is executed with preemption e= nabled */ # define cant_sleep() \ - do { __cant_sleep(__FILE__, __LINE__, 0); } while (0) + do { __cant_sleep(__FILE__, __LINE__); } while (0) # define sched_annotate_sleep() (current->task_state_change =3D 0) =20 /** diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..aa116daf21bd 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9199,7 +9199,7 @@ void __might_resched(const char *file, int line, unsi= gned int offsets) } EXPORT_SYMBOL(__might_resched); =20 -void __cant_sleep(const char *file, int line, int preempt_offset) +void __cant_sleep(const char *file, int line) { static unsigned long prev_jiffy; =20 @@ -9209,7 +9209,7 @@ void __cant_sleep(const char *file, int line, int pre= empt_offset) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; =20 - if (preempt_count() > preempt_offset) + if (preempt_count()) return; =20 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E9AFF37DEA3 for ; Fri, 31 Jul 2026 20:30:54 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529857; cv=none; b=NFqdC/v8n3IOPzXrB3uK5g3AeDsTvF6nKKDi8KA2cswNMsdeTSJ4KtgCi2uQ/lnYYsLI4Ja4GryjtS8Aq/3FYOLr8slQw/EFp+XGAlQPuqeyB2fkpcok/BRz75bLkw8CUT3ufLRanh6/UBLkdEaP2tbQR2eZf2OvJe1Jkr6OUS4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529857; c=relaxed/simple; bh=DDKCSBmWdh69FjGpE3KtoKKl8diHs5FB1tKxxEDIwtY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=OqT7c17G3bVEpv+e+5Cr2xLUrJ0M67m5gnhhPonxVO3QTeKWAF2yDGW3ispi9ex4D6WhysywCenASh0VIn2eE8oOPI0k/oif6PvaGHycOPdJB/T+K+p2NOka9tjWy5KghkPsOBS7GIAAEEbjqOr/IwWxCY34582RtXLsviBjKCg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=T/hbaPJ4; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="T/hbaPJ4" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7F3BC1F0155E; Fri, 31 Jul 2026 20:30:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529853; bh=XDCFK0Pykwc7yJhmq8+Cu0tsCgs0jRR2+x2RB8Lxd70=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=T/hbaPJ45q7zs2zifUXRStZDCLGs5HlF9QIzPDWbPyb4FWgWR8qGnPf+uJ6/OBDv5 pwUon9jYMwOnQ35PjS7NtzZrcELqeNzvSsT175FVmvYitgoj/7pbpPpD9hxLgyZXH/ at3FXcD/Bwov7n8kMXIc/PUAJo/GWynXtweoNEg0yKT2PdTYn3UiSM2zxIPUUrq0Nv oURy1t1Qtv/eAfCMmMeVd1i84n7FuUJQlYyhKDJC9P7pQQuZvdFUYBoMC0CpnSuRxh jyaIoYpJj/aM8pAqhJ9sjwnV1Nb4BLOHV4EOfC4L7UmlXJmqvqyqmMQk4IlRjn4X5i SJc+LeEk/LKBA== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id AEC11F40077; Fri, 31 Jul 2026 16:30:52 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-06.internal (MEProxy); Fri, 31 Jul 2026 16:30:52 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFIC2VVTLK13uDkALWshPgkkwS9DC5Q8ljHaA42IPRnmgkrxQSeK5HYsDRhN4GhLk 6Lg9dLbHgT0DH+qVaNoW+UUFxptAXHUIdqGwEJPczm0ywmTmF9/CSNIQrGZfaA0/SYDXt9 FCHeSX4ZePBeVl3Top6EjJd42yXsFqp+L0LPuLnabUXIo8fT3yX5hMg5WmOramFVWkXrXb dtjJi0qTpjt7TXVawGUwOyoJ/Q6jP1wNaVQH+6xYGS7UXVpHVglHDWcX7SjmpbiByeHS7C oLMM3JHkGXdP7WkAt31sGvl3IAmYfCODupTHvovIO6+oDBZmEBw2YefLw5AUKyninHXISS fcBtE/oYMBzM/w4XXdH/a5eaaRY1NV57i2omU3LHbqUblqPHtNVtoFNc+U3a0UA2khKoQp ZcuvCggAZ8WU9KZiPN3M5l0Vf5AG8lSohqpQXwbEe+EnT3a3fC3f+hT55Tj8i6/P4aBCZz GZhzJOE9yx1tCkUrpU0MBWhBNvnlF3OhZdTcXtvkpeOmTia5KwxMsi2qrl6EENEkT2QF2A uOWKtM5Ftuo66Ouox10iJQG2RaTEYmCP4nqpafXKZ3B70Z6aTXz3dC3Ks/8/xYuX0uCCss 8osp5gC0z6GKsP6MowYZL7nF8XFW2CpD2ena+A3Fn6b/k1BhhtqnYLX9alJg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:52 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH 09/24] sched: Avoid signed comparison of preempt_count() in __cant_migrate() Date: Fri, 31 Jul 2026 13:30:10 -0700 Message-ID: <20260731203031.13679-10-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently preempt_count() is always a non-negative int on all archs (PREEMPT_NEED_RESCHED archs will mask out the MSB when return preempt_count()), hence the checking in __cant_migrate() is in fact just checking whether preempt_count() is 0 or not. In a future change, we are going to use all the 32 bits of preempt_count(), which would make negative int values possible from preempt_count(). Therefore convert the "> 0" comparison into a zero checking to prepare for the future change. No functional changes are intended. Signed-off-by: Boqun Feng --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index aa116daf21bd..9b3f1764fa9e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9241,7 +9241,7 @@ void __cant_migrate(const char *file, int line) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; =20 - if (preempt_count() > 0) + if (preempt_count()) return; =20 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8C80A37C902 for ; Fri, 31 Jul 2026 20:30:55 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529858; cv=none; b=tMo4jzaw/8mJ6RNN/7fJ7lbKjhP2I2ZPzqK1qpxCDwd2WFDXJQoSBPMzgFEzpaCs+Zn6unZfrMp+1dWB7pzp/uGx30jmzJC7gZfp24f1CcjSIk4kGqtLXgjp5ZGXzya+3LYvjPRjGMKvXQspaA7DGmucWCh/Joh96TXfv3fdHAI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529858; c=relaxed/simple; bh=wDgCVjJ/tA3qX/HeiFMz1nTp8TMLkeOU11f9oh74hBs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=cu1uMUuACH6J7av2uQwOG6ZTOLsUselnrsjU/uXvS+GSfp5eEd/80bUgdsSWxOCqpu+dGPICdnCgMbLpupYPHys4u4pmCvfpQSPBYqmUaQr1wKlNF6ZBqE+Y//gMQ5QnB5Qk5vrCll3jMxzfBti0lHbzGsveZrrvft1zZlVjtt0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=NFCAFdPx; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="NFCAFdPx" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E78691F00ADE; Fri, 31 Jul 2026 20:30:54 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529855; bh=IEhLMS7nxZRwSg2vdpqlNZceshQyW6DjfMg9v8g3lYE=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=NFCAFdPxhdQh8BRnApTEFVGAXv4YfzWShQEXyhNJLpSbL4LVokBGlGo3YZs14pPfC D4i6J8wzH4fZI6utppSDDVW/OKEoysuhKzsaVlZQYzyrfNCUvgHUiIhrErVpVkiVjb 797s6G9gh3458cRy32tWgGVudlMW23wXTfFT73qk+iijb4qBWe8SWHq+SPJ9Rl3mN1 gY/4C/z2zeQ9hYIAzZfWTjy2HegCnCZ4v0JUvu7RJDJqkYjBWpi8Ev+3AXz8m7GJCh +brxbmfvc3X65waodSXFBfGAQXqvDb8SaRjQVTUBT57j7LNrGNkENb5IXcRfx6AgjM Cik8xlerX00JA== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 1EC57F4007A; Fri, 31 Jul 2026 16:30:54 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-05.internal (MEProxy); Fri, 31 Jul 2026 16:30:54 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFIC2VVTLK13uDkALWshPgkkwS9DC5Q8ljHaA42IPRnmgkrxQSeK5HYsDRhN4GhLk 6Lg9dLbHgT0DH+qVaNoW+UUFxptAXHUIdqGwEJPczm0ywmTmF9/CSNIQrGZfaA0/SYDXt9 FCHeSX4ZePBeVl3Top6EjJd42yXsFqp+L0LPuLnabUXIo8fT3yX5hMg5WmOramFVWkXrXb dtjJi0qTpjt7TXVawGUwOyoJ/Q6jP1wNaVQH+6xYGS7UXVpHVglHDWcX7SjmpbiByeHS7C oLMM3JHkGXdP7WkAt31sGvl3IAmYfCODupTHvovIO6+oDBZmEBw2YefLw5AUKyninHXIXb XzY00AcIvzX8Hr7Ddyn34ihLwau265zq/BOJSu2D80uZkRDGelLx79QX7cWFsiNnULjQtd +2o+HjfoKa9NPxNgeT1E+FmButCt60mA37kWV0NyX3kxtKSAFwC4sxB9sJ6hmorPm7Ftgy QQSeoyIcd5SmPXRiJkJwaeRf+nTDf0Ezs6j3n1IUpODt+59Gubrv9Qns2ff/5xNOGoIBY1 DgxUdE/X6MapPz/cBPu/F/Xkf8MH6j7h5xRTRFauVURKKoTa8QY6q373qg02fcHHHJX21/ SH2IwBQMLgNwOoqhmfUzQK78d7PQaV/a+W2F24bUrWi3YNvy810EtneAKV0g X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:53 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH 10/24] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Fri, 31 Jul 2026 13:30:11 -0700 Message-ID: <20260731203031.13679-11-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" With the changes that enable preempt count to tracking irq disabling nesting, we don't have enough bits in 32bit preempt count implementation, as a result we move NMI nesting bits out of the 32bit preempt count. However on the architectures that can support 64bit preempt count implementation, we can keep the NMI nesting bits in the 32bit preempt count and avoid maintaining NMI nesting bits out of the same cache line. Therefore HAS_SEPARATE_PREEMPT_RESCHED_BITS is introduced to allow architectures to select this. Note that under this kconfig, preempt count is maintained in a 64bit word however preempt_count() still remains as an int because all the effective bits still fit in (previously we mask out NEED_RESCHED bit in preempt_count()). This should make no functional changes for existing preempt_count() users. Enable this for x86_64 along with the introduction of the Kconfig. [boqun: Undo the __preempt_count_{add,sub}() optimization in 32bit preempt count since it may introduce {over,under}flow] Originally-by: Peter Zijlstra Signed-off-by: Boqun Feng --- arch/x86/Kconfig | 1 + arch/x86/include/asm/preempt.h | 55 +++++++++++++++++++++++----------- arch/x86/kernel/cpu/common.c | 2 +- include/linux/hardirq.h | 48 +++++++++++++++++++++-------- include/linux/preempt.h | 20 +++++++------ kernel/Kconfig.preempt | 4 +++ kernel/sched/core.c | 12 ++++++-- kernel/softirq.c | 6 ++++ lib/locking-selftest.c | 2 +- 9 files changed, 107 insertions(+), 43 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..6a7067d20a6a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -326,6 +326,7 @@ config X86 select USER_STACKTRACE_SUPPORT select HAVE_ARCH_KCSAN if X86_64 select PROC_PID_ARCH_STATUS if PROC_FS + select HAS_SEPARATE_PREEMPT_RESCHED_BITS if X86_64 && PREEMPT_COUNT select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX select FUNCTION_ALIGNMENT_16B if X86_64 || X86_ALIGNMENT_16 select FUNCTION_ALIGNMENT_4B diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 1220656f3370..12353eeebc52 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -7,10 +7,20 @@ =20 #include =20 -DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count); +DECLARE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count); =20 -/* We use the MSB mostly because its available */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* + * We use the MSB for PREEMPT_NEED_RESCHED mostly because it is available. + */ +#define PREEMPT_NEED_RESCHED (~(((unsigned long)-1L) >> 1)) + +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +#define __pc_dec "decq" +#define __pc_op(op, ...) raw_cpu_##op##_8(__VA_ARGS__) +#else +#define __pc_dec "decl" +#define __pc_op(op, ...) raw_cpu_##op##_4(__VA_ARGS__) +#endif =20 /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -24,18 +34,26 @@ DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count); */ static __always_inline int preempt_count(void) { - return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED; + return __pc_op(read, __preempt_count) & ~PREEMPT_NEED_RESCHED; } =20 -static __always_inline void preempt_count_set(int pc) +/* + * unsigned long preempt count parameter works for both 32bit and 64bit ca= ses: + * + * - For 32bit, "int" (the return of preempt_count()) and "unsigned long" = have + * the same size. + * - For 64bit, the effective bits of a preempt count sits in 32bit, and we + * reserve the NEED_RESCHED bit from the old count. + */ +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; =20 - old =3D raw_cpu_read_4(__preempt_count); + old =3D __pc_op(read, __preempt_count); do { new =3D (old & PREEMPT_NEED_RESCHED) | (pc & ~PREEMPT_NEED_RESCHED); - } while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new)); + } while (!__pc_op(try_cmpxchg, __preempt_count, &old, new)); } =20 /* @@ -58,17 +76,17 @@ static __always_inline void preempt_count_set(int pc) =20 static __always_inline void set_preempt_need_resched(void) { - raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED); + __pc_op(and, __preempt_count, ~PREEMPT_NEED_RESCHED); } =20 static __always_inline void clear_preempt_need_resched(void) { - raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED); + __pc_op(or, __preempt_count, PREEMPT_NEED_RESCHED); } =20 static __always_inline bool test_preempt_need_resched(void) { - return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED); + return !(__pc_op(read, __preempt_count) & PREEMPT_NEED_RESCHED); } =20 /* @@ -77,22 +95,22 @@ static __always_inline bool test_preempt_need_resched(v= oid) =20 static __always_inline void __preempt_count_add(int val) { - raw_cpu_add_4(__preempt_count, val); + __pc_op(add, __preempt_count, val); } =20 static __always_inline void __preempt_count_sub(int val) { - raw_cpu_add_4(__preempt_count, -val); + __pc_op(add, __preempt_count, -val); } =20 static __always_inline int __preempt_count_add_return(int val) { - return raw_cpu_add_return_4(__preempt_count, val); + return __pc_op(add_return, __preempt_count, val); } =20 static __always_inline int __preempt_count_sub_return(int val) { - return raw_cpu_add_return_4(__preempt_count, -val); + return __pc_op(add_return, __preempt_count, -val); } =20 /* @@ -102,7 +120,7 @@ static __always_inline int __preempt_count_sub_return(i= nt val) */ static __always_inline bool __preempt_count_dec_and_test(void) { - return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e, + return GEN_UNARY_RMWcc(__pc_dec, __my_cpu_var(__preempt_count), e, __percpu_arg([var])); } =20 @@ -111,7 +129,7 @@ static __always_inline bool __preempt_count_dec_and_tes= t(void) */ static __always_inline bool should_resched(int preempt_offset) { - return unlikely(raw_cpu_read_4(__preempt_count) =3D=3D preempt_offset); + return unlikely(__pc_op(read, __preempt_count) =3D=3D preempt_offset); } =20 #ifdef CONFIG_PREEMPTION @@ -158,4 +176,7 @@ do { \ =20 #endif /* PREEMPTION */ =20 +#undef __pc_op +#undef __pc_dec + #endif /* __ASM_PREEMPT_H */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3df21d26460..73a6d9f6a78e 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2236,7 +2236,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, curren= t_task) =3D &init_task; EXPORT_PER_CPU_SYMBOL(current_task); EXPORT_PER_CPU_SYMBOL(const_current_task); =20 -DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) =3D INIT_PREEMPT_COUNT; +DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) =3D INIT_PREEMPT_= COUNT; EXPORT_PER_CPU_SYMBOL(__preempt_count); =20 DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) =3D TOP_= OF_INIT_STACK; diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 8d4895531a45..b5dc907274db 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -10,8 +10,6 @@ #include #include =20 -DECLARE_PER_CPU(unsigned int, nmi_nesting); - extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); =20 @@ -94,6 +92,38 @@ void irq_exit_rcu(void); #define arch_nmi_exit() do { } while (0) #endif =20 +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +static __always_inline void __preempt_count_nmi_enter(void) +{ + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); +} + +static __always_inline void __preempt_count_nmi_exit(void) +{ + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); +} +#else +DECLARE_PER_CPU(unsigned int, nmi_nesting); + +#define __preempt_count_nmi_enter() \ + do { \ + __preempt_count_add(HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ + __this_cpu_inc(nmi_nesting); \ + preempt_count_set(preempt_count() | NMI_MASK); \ + } while (0) + +#define __preempt_count_nmi_exit() \ + do { \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + if (!__this_cpu_dec_return(nmi_nesting)) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ + } while (0) + +#endif + + /* * NMI vs Tracing * -------------- @@ -110,18 +140,14 @@ void irq_exit_rcu(void); do { \ lockdep_off(); \ arch_nmi_enter(); \ - /* Maximum NMI nesting is 15. */ \ - BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ - __this_cpu_inc(nmi_nesting); \ - __preempt_count_add(HARDIRQ_OFFSET); \ - preempt_count_set(preempt_count() | NMI_MASK); \ + __preempt_count_nmi_enter(); \ } while (0) =20 #define nmi_enter() \ do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ - ct_nmi_enter(); \ + ct_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ @@ -129,12 +155,8 @@ void irq_exit_rcu(void); =20 #define __nmi_exit() \ do { \ - unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(HARDIRQ_OFFSET); \ - nesting =3D __this_cpu_dec_return(nmi_nesting); \ - if (!nesting) \ - preempt_count_set(preempt_count() & ~NMI_MASK); \ + __preempt_count_nmi_exit(); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 33fc4c814a9f..87d5367f986c 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -30,18 +30,20 @@ * NMI nesting depth is tracked in a separate per-CPU variable * (nmi_nesting) to save bits in preempt_count. * - * PREEMPT_MASK: 0x000000ff - * SOFTIRQ_MASK: 0x0000ff00 - * HARDIRQ_DISABLE_MASK: 0x00ff0000 - * HARDIRQ_MASK: 0x0f000000 - * NMI_MASK: 0x10000000 - * PREEMPT_NEED_RESCHED: 0x80000000 + * 32bit HAS_SEPARATE_PREEMPT_RESCHED_BITS + * + * PREEMPT_MASK: 0x000000ff 0x00000000000000ff + * SOFTIRQ_MASK: 0x0000ff00 0x000000000000ff00 + * HARDIRQ_DISABLE_MASK: 0x00ff0000 0x0000000000ff0000 + * HARDIRQ_MASK: 0x0f000000 0x000000000f000000 + * NMI_MASK: 0x10000000 0x00000000f0000000 + * PREEMPT_NEED_RESCHED: 0x80000000 0x8000000000000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 1 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BIT= S)) =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) @@ -116,8 +118,8 @@ static __always_inline unsigned char interrupt_context_= level(void) * preempt_count() is commonly implemented with READ_ONCE(). */ =20 -#define nmi_count() (preempt_count() & NMI_MASK) -#define hardirq_count() (preempt_count() & HARDIRQ_MASK) +#define nmi_count() (preempt_count() & NMI_MASK) +#define hardirq_count() (preempt_count() & HARDIRQ_MASK) #ifdef CONFIG_PREEMPT_RT # define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK) # define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | sof= tirq_count()) diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 88c594c6d7fc..35f546a042b1 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK config PREEMPT_COUNT bool =20 +config HAS_SEPARATE_PREEMPT_RESCHED_BITS + bool + depends on PREEMPT_COUNT && 64BIT + config PREEMPTION bool select PREEMPT_COUNT diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9b3f1764fa9e..6d88343c3bad 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5973,8 +5973,13 @@ void preempt_count_add(int val) #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? + * + * Cannot detect underflow based on the current preempt_count() value + * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes= all 32 + * bits. */ - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) && + DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) return; #endif __preempt_count_add(val); @@ -6006,7 +6011,10 @@ void preempt_count_sub(int val) /* * Underflow? */ - if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) + unsigned int uval =3D val; + unsigned int pc =3D preempt_count(); + + if (DEBUG_LOCKS_WARN_ON(pc - uval > pc)) return; /* * Is the spinlock portion underflowing? diff --git a/kernel/softirq.c b/kernel/softirq.c index d1ab1799794c..491136a313db 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -91,7 +91,13 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); DEFINE_PER_CPU(struct interrupt_disable_state, local_interrupt_disable_sta= te); EXPORT_PER_CPU_SYMBOL_GPL(local_interrupt_disable_state); =20 +#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +/* + * Any 32bit architecture that still cares about performance should + * probably ensure this is near preempt_count. + */ DEFINE_PER_CPU(unsigned int, nmi_nesting); +#endif =20 /* * SOFTIRQ_OFFSET usage: diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index bfafe1204c7b..c3d976c801bb 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures; =20 static void dotest(void (*testcase_fn)(void), int expected, int lockclass_= mask) { - int saved_preempt_count =3D preempt_count(); + long saved_preempt_count =3D preempt_count(); #ifdef CONFIG_PREEMPT_RT int saved_mgd_count =3D current->migration_disabled; int saved_rcu_count =3D current->rcu_read_lock_nesting; --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1074037C922; Fri, 31 Jul 2026 20:30:57 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529859; cv=none; b=mM0jNTdEryEnepNQ9RWdfqfBI+RTdAMyldjUTjMcYAYKnCQJUw9P1LeD1i6XlacZEMxyhqUBNKCxfHEy9LnT9es9K1ce+/MfuGOxwlM2pre0/6VSrFBsiaYVPlGQNK9muuUI/Rj1p4+neWkhpaTAJivaYXEQkrx7quBohw+ODTo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529859; c=relaxed/simple; bh=yofG9RF80NQjUze7PJ8mZv/CsasdVwXyT7Se9I6pjwg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=sbWtV1s6MUAnrrtDzKt+L/oIk9Qn0pTO6KP2R1ApblbNWqyau6zAvE/ARq7kn1hmpAj8pyCoNSo9qulVJ7OB5PbGuApaZ/mXSum6bvag15IBOU2Mdpw6E3fw0hCcIWC2vrE5zlXSEBggldNVq85+Ci+FSP8iedahz1ZSP29Fo0U= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=IyoP+f18; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="IyoP+f18" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 5BBAE1F00ACA; Fri, 31 Jul 2026 20:30:56 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529856; bh=9stI0iersWJs4iyjIhDsa1iyfFyNlTFSbpf6Xl0dIso=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=IyoP+f18s9yEW8PiSUAazUwEcJG2FyZuRtsBtWlqM4KdYUhe75FdTl6g4lhaDUGPH 2PbgN6aZ73KKO/dGruTA6LJReT/jM8Jb9zGxd2xrjTFwWMJMg3gYBQgsWalKo/8CLo jC+LbmhmnLIigmGUD5jhlc9W/G92SX/51lNnS2QxaW3rMgECjbMvELAgM5LLEku4SM 25GE/HaXLdt5YP8X+LvmH3NyVDE+N25uxzhaxW/Qk6nH1yDFbJSs+EPfMjVCAapEdY OJrD2lfTMsJTABE1/ONuTFOMPf6R5JEy1ycIA+0OX++x6xBxrKxekOn+cszSjEAsjJ U8/8h4mRvrFUw== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 89703F40077; Fri, 31 Jul 2026 16:30:55 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-05.internal (MEProxy); Fri, 31 Jul 2026 16:30:55 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFIC2VVTLK13uDkALWshPgkkwS9DC5Q8ljHaA42IPRnmgkrxQSeK5HYsDRhN4GhLk 6Lg9dLbHgT0DH+qVaNoW+UUFxptAXHUIdqGwEJPczm0ywmTmF9/CSNIQrGZfaA0/SYDXt9 FCHeSX4ZePBeVl3Top6EjJd42yXsFqp+L0LPuLnabUXIo8fT3yX5hMg5WmOramFVWkXrXb dtjJi0qTpjt7TXVawGUwOyoJ/Q6jP1wNaVQH+6xYGS7UXVpHVglHDWcX7SjmpbiByeHS7C oLMM3JHkGXdP7WkAt31sGvl3IAmYfCODupTHvovIO6+oDBZmEBw2YefLw5AUKyninHXIXy 0ZXO16SKIuDo5nyOIq7d02biZVmbCBmXHXP05KiV1qIcjo5kZWDDRtfU2ASUo8X7LGFrIt Tpix6SzsCRWNcrMnGj2rB1DUUCKAM62wzNVK4Yj95qqbVu2uELV5lMIkIIptJKJJjrI2oO 0Aatr1LV4heqtjommZRtFmfYM4czdD+mNtbkrRWol37ObifjxcHzaXXzKIyrc+8OuxBarn dbi6PPzLgBHZGTQWAutr7TSuZcpwvdrEGtzH8A68YThDJgA0kmk2WLvjQIkgqjZBBP4iuP hv9pil5GY75sR6yTDxqgWm1CJwPEKUEqxqOv+IjzfQjjMvxzC1leg5J7ds7A X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:54 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH 11/24] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Fri, 31 Jul 2026 13:30:12 -0700 Message-ID: <20260731203031.13679-12-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" ARM64 already uses 64bit preempt count and the need reschedule bit is maintained in a separate 32bit than the preempt count. Therefore preempt count has enough bits to represent 16 level of NMI nesting, hence enable it for ARM64. This saves a per-CPU variable and additional instructions in the NMI path. Signed-off-by: Boqun Feng --- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..349c3533cd1e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -247,6 +247,7 @@ config ARM64 select PCI_SYSCALL if PCI select POWER_RESET select POWER_SUPPLY + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 981F63822AA; Fri, 31 Jul 2026 20:30:58 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529860; cv=none; b=WT972sDmNabxBqIWW3CU/hVOn3ieEj1ynee/79IGsAdNGzzb8yY00UHOxaQR/ozLXJPsvtcM5fnTVn/oJW4DxATLQbGyZ8dO8GDLMUzP1qiGLGE/hnAHnxTA5i86NWOVyfDKd/kcG3VGWVsgmQgVZ0FmyiwCFcpmFgY7yUQ0Nw4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529860; c=relaxed/simple; bh=raWQqz9gQpyQdZeeG6jtU2j//DLg/OLByTkuNHS7FRQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=hPOAuPBMT5zVm9PhddnpsjgiLui6BUYPHaT1oy3ZQEzlR+UlkLtthmKQS/gsNtgPRYjsArbbdd4R7Obn8rIzGrPUDAzScCpcj1wIPL1r9gVi1Fa0wBcKLTYBXhYYYW5t3TJQ+kzIEnbCyPNitTAszM29OfrRE7d/PD15P53oMkc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=HnnDe5el; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="HnnDe5el" Received: by smtp.kernel.org (Postfix) with ESMTPSA id DD5201F00ACF; Fri, 31 Jul 2026 20:30:57 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529858; bh=gPacRWUE8d9BStFdwgFA5J4tq/QViHyWDEzGTjWCNxE=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=HnnDe5elH0H6FsXTKSqHqpXL39HuCasqI0BUcDBIKpT3FKWLhlvydva+KAw+dmY4X 3/GFSI8HuN3GAcN5xhYIW14HVs845w73TGhdNqKRptxkVADOG0QRwnUDIE05Brpxz7 4M+Jbh18YAbpHVqJRixztj38x2h1wCPqsA04oovxzArecfyDJten4R1P9zyyxksGxe MIALSxxPSKvA92mZnZcBsOQEHioXfpeX62rZZCOLi09QF3etxFwQQvULbVGA/o7pQd +9S5cVIw3aVN8AvFrs8vIkaAoWIGl2CIZLKf8ujzHh7XQk6o5gG48XimSMOfbo1+uj 7z2nl/uou+OWA== Received: from phl-compute-12.internal (phl-compute-12.internal [10.202.2.52]) by mailfauth.phl.internal (Postfix) with ESMTP id 17BC6F40077; Fri, 31 Jul 2026 16:30:57 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-12.internal (MEProxy); Fri, 31 Jul 2026 16:30:57 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdRdBIF8ZH/aPs2g1lRw7GATpB3kPZdHQ9SRq3TCfYELIaqSZraFLrXlLQvfZ+Zw L2FnPfSJd952oWr+y28mIfAsYcvh2g32U0J8iwTV49f0k9Q8hXOe/2BgVhInUEnjhmC2lX 03vYTAlN0bq9NjF03uLbRgPGL4766SVn/XbgM9h+YDdXpmZY+TYoTPsYcygLFMbtetKCiX MKKxau189OoX/tBJGac9kkLntk5lYAb6ORV5iOKO0UE2KpFdBfn5cZVhjAZINCJF+DQAjd IJX9TqJvKbTOKjLKBWundtWMSfFqKbkgkDYizysOgKGZkbbhOs1gMmPcuy0Gf54a13j8l7 7gCw/KYUbI4D9PNQkOz6D7Cds5w/Kng7H6GTcf7i1M4HyhTC/n43N+aJ9bqxSGYASrFjZE vRjX0J3uIxYM68WkNnFTHr648ylqW/1DqCHb74Nb/fCLMMoONWrkiP3YZ5P0EUyy0Or5BP 7pAQdiexdo98vGTYLhBdWNOXOYjewm3c9o+H/sEssZmK1zLAZYZDXtp7J7h4SHYUHlR6TI Zd4wBtuvHHhqGGfOAdIjux71buWJkJDul1ZHznPa7CjLM/n4pyc3LQtaD7Q0byQABRc5Lu eJmRdljxBP6rQUI+Gah8DtbWaAsxAzwchKsQ2llp+ugKzKfiFXH/870EpnXg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:56 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Heiko Carstens Subject: [PATCH 12/24] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Fri, 31 Jul 2026 13:30:13 -0700 Message-ID: <20260731203031.13679-13-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Heiko Carstens Convert s390's preempt_count to 64 bit, and change the preempt primitives accordingly. Signed-off-by: Heiko Carstens Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260509181249.16281C67-hca@linux.ibm.com --- arch/s390/Kconfig | 1 + arch/s390/include/asm/lowcore.h | 13 +++++++---- arch/s390/include/asm/preempt.h | 41 +++++++++++++++------------------ 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..378fcd2b6181 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -273,6 +273,7 @@ config S390 select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select PCI_QUIRKS if PCI + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcor= e.h index 3b3ecc647993..5cef215d30e7 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -160,10 +160,15 @@ struct lowcore { /* SMP info area */ __u32 cpu_nr; /* 0x03a0 */ __u32 softirq_pending; /* 0x03a4 */ - __s32 preempt_count; /* 0x03a8 */ - __u32 spinlock_lockval; /* 0x03ac */ - __u32 spinlock_index; /* 0x03b0 */ - __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ + union { + struct { + __u32 need_resched; /* 0x03a8 */ + __u32 count; /* 0x03ac */ + } preempt; + __u64 preempt_count; /* 0x03a8 */ + }; + __u32 spinlock_lockval; /* 0x03b0 */ + __u32 spinlock_index; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ __u8 percpu_register; /* 0x03c0 */ __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preemp= t.h index 0a25d4648b4c..1d5e4d7e9e1b 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -8,11 +8,8 @@ #include #include =20 -/* - * Use MSB so it is possible to read preempt_count with LLGT which - * reads the least significant 31 bits with a single instruction. - */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */ +#define PREEMPT_NEED_RESCHED 0x8000000000000000UL =20 /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -26,25 +23,25 @@ */ static __always_inline int preempt_count(void) { - unsigned long lc_preempt, count; + unsigned long lc_preempt; + int count; =20 - BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) !=3D sizeof(int)= ); - lc_preempt =3D offsetof(struct lowcore, preempt_count); + lc_preempt =3D offsetof(struct lowcore, preempt.count); /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */ asm_inline( - ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n", - "llgt %[count],%[offalt](%%r0)\n", + ALTERNATIVE("ly %[count],%[offzero](%%r0)\n", + "ly %[count],%[offalt](%%r0)\n", ALT_FEATURE(MFEATURE_LOWCORE)) : [count] "=3Dd" (count) : [offzero] "i" (lc_preempt), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS), - "m" (((struct lowcore *)0)->preempt_count)); + "m" (((struct lowcore *)0)->preempt.count)); return count; } =20 -static __always_inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; =20 old =3D READ_ONCE(get_lowcore()->preempt_count); do { @@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc) =20 static __always_inline void set_preempt_need_resched(void) { - __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_cou= nt); } =20 static __always_inline void clear_preempt_need_resched(void) { - __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count= ); } =20 static __always_inline bool test_preempt_need_resched(void) @@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val) =20 lc_preempt =3D offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("asi %[offzero](%%r0),%[val]\n", - "asi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n", + "agsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (val), @@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val) return; } } - __atomic_add(val, &get_lowcore()->preempt_count); + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } =20 static __always_inline void __preempt_count_sub(int val) @@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_t= est(void) =20 lc_preempt =3D offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n", - "alsi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n", + "algsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "=3D@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (-1), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS)); return (cc =3D=3D 0) || (cc =3D=3D 2); #else - return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count); + return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_= count); #endif } =20 @@ -141,7 +138,7 @@ static __always_inline bool should_resched(int preempt_= offset) =20 static __always_inline int __preempt_count_add_return(int val) { - return val + __atomic_add(val, &get_lowcore()->preempt_count); + return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } =20 static __always_inline int __preempt_count_sub_return(int val) --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B8629383994; Fri, 31 Jul 2026 20:30:59 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529861; cv=none; b=iRvsmdPYFaDU2wtqwI9LqxzYYEhe7VJhMpdjcc7y12fxmTCxqxu5fj3Lq8ZqVlZwtpogx/axbvAar6XDKdktrMKU8ZT8mrK1Q3z3kYEo4pnzm04debiyxepcNxqy99Ym/emyORvHdkAdmqH49EEk8Y6TX7GXCBdKmGOg8y6PQNI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529861; c=relaxed/simple; bh=VVUEAuv0knUbJ18uPwAHlKSyW+OJF8lfXoMGXU2hiFI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Vp+lGysYlNrCYwl3KDr/gKcH/5wr+phdK1wiMYwZN7CFcUn1T+n7M+WlqZRHfloKBahNxWGojzel5mmoPThnjTp10WjSwDmqrLne6fpwpZb5Lz8l9kdjPeJ3RAhP+0X+AN/XmDymogoJ4yPIjjzJei9pp5KUvyK0H4b29i/7bYY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=YfKUXJj5; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="YfKUXJj5" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 5F6931F00AC4; Fri, 31 Jul 2026 20:30:59 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529859; bh=r3ggz2ubbYMxXvgwctlCQHM16FI4AcWSFWoEFEQ90fI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=YfKUXJj5Dj+nZptsxE30sKDPB+LmBKwUp3/t8e8wxOCK3akju2yx95Qtnf94/Bd2E vFCaSVPUHqGUGG8vUedOxHyjcbA9VVJl9s3Uwx5fXanxkiVwxnLsl64Te9jcCKUiqv 7eSJtthUbUoUrThV3xXbc0jv2BhkH2cgpHVSFE+q+MVIePTa4k/ikwv8WoGoCdB6Mk hkEaKhuLECaDHFcF6n3llb+PIjX8+wx8x3OklPCXGWaaVVsbQxMEO/HKfSoUN9lI12 Jvc8UifJaApJpoMpvD4jiowj5PDcnJi0XWyhg8qKzI3waui1SHc19tetmiXrE52ML3 nqyirdxLRSxRQ== Received: from phl-compute-01.internal (phl-compute-01.internal [10.202.2.41]) by mailfauth.phl.internal (Postfix) with ESMTP id 8F642F4007A; Fri, 31 Jul 2026 16:30:58 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-01.internal (MEProxy); Fri, 31 Jul 2026 16:30:58 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFIC2VVTLK13uDkALWshPgkkwS9DC5Q8ljHaA42IPRnmgkrxQSeK5HYsDRhN4GhLk 6Lg9dLbHgT0DH+qVaNoW+UUFxptAXHUIdqGwEJPczm0ywmTmF9/CSNIQrGZfaA0/SYDXt9 FCHeSX4ZePBeVl3Top6EjJd42yXsFqp+L0LPuLnabUXIo8fT3yX5hMg5WmOramFVWkXrXb dtjJi0qTpjt7TXVawGUwOyoJ/Q6jP1wNaVQH+6xYGS7UXVpHVglHDWcX7SjmpbiByeHS7C oLMM3JHkGXdP7WkAt31sGvl3IAmYfCODupTHvovIO6+oDBZmEBw2YefLw5AUKyninHXIng ttqFP0fUYb8AmIHQT5N5eED5S04aSy11dx3DqVX6Kij9bVSlz1RplV3HnpWZfKbZYyKTEK Qcb4g9+wE6Vw6aTEeGnmBpNyRpfw7+iYVYmP/OejSvPLzF9DQMgeHyEoTFUEp6ZUB6US2l ruYDrl0LDy0kjiHqyijbiEUw1CjwyrXBFNZXb7zQB84L8X+Jlbl3rs7QI/MKq5Z47XXhXS 0nuZlTfdmcGiEdUMOJkycPTToZoQ9B/yuHOANP6J5q8IsuI8Co3kuqu+kEWl6r+ogf0D8D lPgUjx17C5DUXIsoBUi3xWMS7D8nZgKop/aWNuOqxsbEWr3wp3FxjflaG/gg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:57 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH 13/24] irq: Optimize reschedule check in local_interrupt_enable() Date: Fri, 31 Jul 2026 13:30:14 -0700 Message-ID: <20260731203031.13679-14-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In local_interrupt_enable(), we could avoid re-reading preempt count because of should_resched() by using the result from hardirq_disable_exit(), however this means __preempt_count_add_return() and __preempt_count_sub_return() need to return all the preempt count bits (including the PREEMPT_NEED_RESCHED bit), since the only user of __preempt_count_{add,sub}_return() is hardirq_disable_{enter,exit}(), hence make them return "unsigned long" to optimize this. Signed-off-by: Boqun Feng --- arch/arm64/include/asm/preempt.h | 12 ++++++------ arch/s390/include/asm/preempt.h | 4 ++-- arch/x86/include/asm/preempt.h | 4 ++-- include/asm-generic/preempt.h | 4 ++-- include/linux/interrupt_rc.h | 31 ++++++++++++++++++++----------- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/pree= mpt.h index 0dd8221d1bef..e9f597d87413 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -55,20 +55,20 @@ static inline void __preempt_count_sub(int val) WRITE_ONCE(current_thread_info()->preempt.count, pc); } =20 -static inline int __preempt_count_add_return(int val) +static inline unsigned long __preempt_count_add_return(int val) { - u32 pc =3D READ_ONCE(current_thread_info()->preempt.count); + u64 pc =3D READ_ONCE(current_thread_info()->preempt_count); pc +=3D val; - WRITE_ONCE(current_thread_info()->preempt.count, pc); + WRITE_ONCE(current_thread_info()->preempt_count, pc); =20 return pc; } =20 -static inline int __preempt_count_sub_return(int val) +static inline unsigned long __preempt_count_sub_return(int val) { - u32 pc =3D READ_ONCE(current_thread_info()->preempt.count); + u64 pc =3D READ_ONCE(current_thread_info()->preempt_count); pc -=3D val; - WRITE_ONCE(current_thread_info()->preempt.count, pc); + WRITE_ONCE(current_thread_info()->preempt_count, pc); =20 return pc; } diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preemp= t.h index 1d5e4d7e9e1b..d0021b979a5d 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -136,12 +136,12 @@ static __always_inline bool should_resched(int preemp= t_offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) =3D=3D preempt_of= fset); } =20 -static __always_inline int __preempt_count_add_return(int val) +static __always_inline unsigned long __preempt_count_add_return(int val) { return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } =20 -static __always_inline int __preempt_count_sub_return(int val) +static __always_inline unsigned long __preempt_count_sub_return(int val) { return __preempt_count_add_return(-val); } diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 12353eeebc52..fc1a2799990a 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -103,12 +103,12 @@ static __always_inline void __preempt_count_sub(int v= al) __pc_op(add, __preempt_count, -val); } =20 -static __always_inline int __preempt_count_add_return(int val) +static __always_inline unsigned long __preempt_count_add_return(int val) { return __pc_op(add_return, __preempt_count, val); } =20 -static __always_inline int __preempt_count_sub_return(int val) +static __always_inline unsigned long __preempt_count_sub_return(int val) { return __pc_op(add_return, __preempt_count, -val); } diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index c8683c046615..7629e23102d1 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -59,14 +59,14 @@ static __always_inline void __preempt_count_sub(int val) *preempt_count_ptr() -=3D val; } =20 -static __always_inline int __preempt_count_add_return(int val) +static __always_inline unsigned long __preempt_count_add_return(int val) { *preempt_count_ptr() +=3D val; =20 return *preempt_count_ptr(); } =20 -static __always_inline int __preempt_count_sub_return(int val) +static __always_inline unsigned long __preempt_count_sub_return(int val) { *preempt_count_ptr() -=3D val; =20 diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h index dd4444c61330..c044dc395452 100644 --- a/include/linux/interrupt_rc.h +++ b/include/linux/interrupt_rc.h @@ -27,7 +27,7 @@ DECLARE_PER_CPU(struct interrupt_disable_state, local_int= errupt_disable_state); static inline void local_interrupt_disable(void) { unsigned long flags; - int new_count; + unsigned long new_count; =20 WARN_ON_ONCE(in_nmi()); =20 @@ -41,9 +41,25 @@ static inline void local_interrupt_disable(void) } } =20 +#ifdef CONFIG_PREEMPTION +static inline void local_interrupt_enable_reched(unsigned long pc) +{ + if (pc) + return; + /* No PREEMPT_NEED_RESCHED bit? Check tif_need_resched() */ +#ifndef PREEMPT_NEED_RESCHED + if (!tif_need_resched()) + return; +#endif + __preempt_schedule(); +} +#else +static inline void local_interrupt_enable_reched(unsigned long pc) {} +#endif + static inline void local_interrupt_enable(void) { - int new_count; + unsigned long new_count; =20 new_count =3D hardirq_disable_exit(); =20 @@ -52,15 +68,8 @@ static inline void local_interrupt_enable(void) =20 flags =3D raw_cpu_read(local_interrupt_disable_state.flags); local_irq_restore(flags); - /* - * TODO: re-read preempt count can be avoided, but it needs - * should_resched() taking another parameter as the current - * preempt count - */ -#ifdef CONFIG_PREEMPTION - if (should_resched(0)) - __preempt_schedule(); -#endif + + local_interrupt_enable_reched(new_count); } } =20 --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5933A386429; Fri, 31 Jul 2026 20:31:01 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529863; cv=none; b=iONvItI/5CQcUE+82ruZxUck6/oSEge5tbbn/iXamnweJYHycqikRv4ei5s0mBUV+zRRuCoTG7xN9KDgBPkXCFCLZdjFy53YLFNYNSNrMYEJLmrl67mSv/DxqWJWPMTP4ogr1rZb+DlxndRaBYq8IAjBAGNqJey+0k90li07pHM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529863; c=relaxed/simple; bh=Zq44MwbXDOcOoz+Qfz1RwXEda/JiiBqyhD5TdIogL4c=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=WnukYabG536P5T/mLYpEbxe/1F81dsKB4zAYTxe8fuPhwFEgVDBtbkWWqs1JX/wG8qURuBTDb1etjxpjM+XpEpj23Ox63L0UHNtG0nVgagsRK2Tn1ImUGSQRTpoHtpgBFpC7FoWoNoBDYflN+L+YlWMaFebQCBp/MgC4cIpJ/Rw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=fL/1Nql0; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="fL/1Nql0" Received: by smtp.kernel.org (Postfix) with ESMTPSA id CE07F1F00ADB; Fri, 31 Jul 2026 20:31:00 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529861; bh=7VuPgpz4qMdyLp6c1Eec/Wedh3mG+ZmpiyvkZ3oX17s=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=fL/1Nql0f+h1DVjDCCq40di0VoaSHDI85+DHRCv/buhikoVM8zGVOb3TPP4fdB0Le Lmq68Er4I8zkfj9CYgqnwj1y3zkjxUZWhOKVSy2d91iEX53pwzgK6Wil5eQc15fXeh oqC1S0+lOhpNJIkq57tzmztScVmD1+w90gvuMcZZbsUlJMCa2lVzDeP+TRNiku5sdT 6S4Z1f7zw6IW4HDeDxOVgcAT2M4dKCv8sVEWDoqaRMbOx1nkd3DSBsvTo824k09oew zU2xFL2Sv/BifOGgNiPX3J6BMCibatMfHe0d3IBpR7FF16Rx2wLGhaynRLHWkVDdvh UJS/a1NWXyx7g== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 0839BF40077; Fri, 31 Jul 2026 16:31:00 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-05.internal (MEProxy); Fri, 31 Jul 2026 16:31:00 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEWpnubMTgFvO8GUjRBMncK13/AJQfdv9iqwcIEXvBf+u3+nTD986gctK6cJQuSpa nAueWKjs/UUb145eUguWg26p1szdm87j0GTWXbRTLgDPU3op3LOZvWryJHZrX0hpe+yTkz u3HIKpzQggIeejA0plSt7O29WoQ9Za8ROWFr88iLOKW68eIlszrjGWvvR/lBHSR0HWQsD9 HRyKpzcT9P0hGBQKgyFUQrBwb1GX9ijM7UyL5Md8UvHEgbjHbf7rmGKBl7KLy8tQRM6vWf gZ478lopPE44VNsCTUy32ohOlAIHeJ6n0VJQvt8VBUd3JafJ8bDUl2vjGZis7/X1s6A2Y3 BB25vPx4EMonDgKFbPi0t0ST0C450mbhzgnQWIbul8dgqVOR6A2e3A7zKB0m5E97k6WexL cuZF/snUBBAuAtbrICmmeNJA0X6mk31Z+omAl0YlLU+JQvhbe3idUEX0YuiCZHuPczzmjh 8MehyfpnzOIo8/WT6kw7i3rcHnEdeOSmv6sew9r2fcFivTy5yDghHVOHFbOIuw71jybDOa oQdp9/Lqorr+rLW0/Sclqe14Bpr7gmRk50NMSiL/opzS/3WBXiCWZNDhfrUp8DGKJLnhl1 wyqtAAipkL6oxpdJINApDQ5m3aagPneirj1xcLJ8F/o7Jxj+VuUyNKeZkGDQ X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:30:59 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Boqun Feng , Benno Lossin , Andreas Hindborg Subject: [PATCH 14/24] rust: Introduce interrupt module Date: Fri, 31 Jul 2026 13:30:15 -0700 Message-ID: <20260731203031.13679-15-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Lyude Paul This introduces a module for dealing with interrupt-disabled contexts, including the ability to enable and disable interrupts along with the ability to annotate functions as expecting that IRQs are already disabled on the local CPU. [Boqun: This is based on Lyude's work on interrupt disable abstraction, I port to the new local_interrupt_disable() mechanism to make it work as a guard type. I cannot even take the credit of this design, since Lyude also brought up the same idea in zulip. Anyway, this is only for POC purpose, and of course all bugs are mine] Signed-off-by: Lyude Paul Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Reviewed-by: Benno Lossin Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260302232154.861916-2-lyude@redhat.com Signed-off-by: Boqun Feng --- rust/helpers/helpers.c | 1 + rust/helpers/interrupt.c | 18 ++++++++ rust/helpers/sync.c | 5 +++ rust/kernel/interrupt.rs | 89 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 5 files changed, 114 insertions(+) create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/kernel/interrupt.rs diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..0d85b5e68ec2 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -65,6 +65,7 @@ #include "irq.c" #include "fs.c" #include "gpu.c" +#include "interrupt.c" #include "io.c" #include "jump_label.c" #include "kunit.c" diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c new file mode 100644 index 000000000000..51b319bd4c00 --- /dev/null +++ b/rust/helpers/interrupt.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ + local_interrupt_disable(); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ + local_interrupt_enable(); +} + +__rust_helper bool rust_helper_irqs_disabled(void) +{ + return irqs_disabled(); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c index 82d6aff73b04..4f474fe847c4 100644 --- a/rust/helpers/sync.c +++ b/rust/helpers/sync.c @@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(str= uct lock_class_key *k) { lockdep_unregister_key(k); } + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ + lockdep_assert_irqs_disabled(); +} diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs new file mode 100644 index 000000000000..667a3bd329fb --- /dev/null +++ b/rust/kernel/interrupt.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Interrupt controls +//! +//! This module allows Rust code to annotate areas of code where local pro= cessor interrupts should +//! be disabled, along with actually disabling local processor interrupts. +//! +//! # =E2=9A=A0=EF=B8=8F Warning! =E2=9A=A0=EF=B8=8F +//! +//! The usage of this module can be more complicated than meets the eye, e= specially surrounding +//! [preemptible kernels]. It's recommended to take care when using the fu= nctions and types defined +//! here and familiarize yourself with the various documentation we have b= efore using them, along +//! with the various documents we link to here. +//! +//! # Reading material +//! +//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/52= 0076) +//! +//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/= preempt-locking.html + +use crate::types::NotThreadSafe; + +/// A guard that represents local processor interrupt disablement on preem= ptible kernels. +/// +/// [`LocalInterruptDisabled`] is a guard type that represents that local = processor interrupts have +/// been disabled on a preemptible kernel. +/// +/// Certain functions take an immutable reference of [`LocalInterruptDisab= led`] in order to require +/// that they may only be run in local-interrupt-disabled contexts on pree= mptible kernels. +/// +/// This is a marker type; it has no size, and is simply used as a compile= -time guarantee that local +/// processor interrupts are disabled on preemptible kernels. Note that no= guarantees about the +/// state of interrupts are made by this type on non-preemptible kernels. +/// +/// # Invariants +/// +/// Local processor interrupts are disabled on preemptible kernels for as = long as an object of this +/// type exists. +pub struct LocalInterruptDisabled(NotThreadSafe); + +/// Disable local processor interrupts on a preemptible kernel. +/// +/// This function disables local processor interrupts on a preemptible ker= nel, and returns a +/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible = kernels, this function is +/// a no-op. +/// +/// **Usage of this function is discouraged** unless you are absolutely su= re you know what you are +/// doing, as kernel interfaces for rust that deal with interrupt state wi= ll typically handle local +/// processor interrupt state management on their own and managing this by= hand is quite error +/// prone. +#[inline] +pub fn local_interrupt_disable() -> LocalInterruptDisabled { + // SAFETY: It's always safe to call `local_interrupt_disable()`. + unsafe { bindings::local_interrupt_disable() }; + + LocalInterruptDisabled(NotThreadSafe) +} + +impl Drop for LocalInterruptDisabled { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariants, a `local_interrupt_disable()` must= be called to create this + // object, hence call the corresponding `local_interrupt_enable()`= is safe. + unsafe { bindings::local_interrupt_enable() }; + } +} + +impl LocalInterruptDisabled { + /// Assume that local processor interrupts are disabled on preemptible= kernels. + /// + /// This can be used for annotating code that is known to be run in co= ntexts where local + /// processor interrupts are disabled on preemptible kernels. It makes= no changes to the local + /// interrupt state on its own. + /// + /// # Safety + /// + /// For the whole life `'a`, local interrupts must be disabled on pree= mptible kernels. This + /// could be a context like for example, an interrupt handler. + #[inline] + pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled { + const ASSUME_DISABLED: &LocalInterruptDisabled =3D &LocalInterrupt= Disabled(NotThreadSafe); + + // Confirm they're actually disabled if lockdep is available + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled(= )` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + ASSUME_DISABLED + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9512af7156df..2ee6c24d39c2 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -82,6 +82,7 @@ pub mod impl_flags; pub mod init; pub mod interop; +pub mod interrupt; pub mod io; pub mod ioctl; pub mod iommu; --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5A72137CD37 for ; Fri, 31 Jul 2026 20:31:03 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529867; cv=none; b=PmTE5nZBuVJN35+7uABWq+xJtIjFjfQcZX515XYRgi6dCLDOSvzm6ysesoGqbvN1EShdifAKa/CiivVHuPgd/94W4pi79riz3WY2SB4CHy4dXeFctW+1aK4CThxqMq2GR7m1UExMFCpxmCzoDEVnDPVHtesgqu/vV/zhErRi12k= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529867; c=relaxed/simple; bh=Csl9FgamiuSComdT0LtWtXb5pp+wvqtY2ufh6bHEU2w=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=WRqSfNM5t2M28QdsaiYz2pp+TDHKSqs5LF0WetvGdo80oeuXHTvJQ8EayQMMbtTZbqvlS3T5lpuwEJwj8sZhmgTrM5TMdKG+r9lCky9sB0WPwBAIy3nblAplyeGZS7fnfQ+laKY97AGiA1qgpwRFRGvEsaI0h6PgG0VYb+feVqA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=T0Qg1Kut; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="T0Qg1Kut" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7577C1F00AC4; Fri, 31 Jul 2026 20:31:02 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529862; bh=3o90b4YG5/hZO8lKJFzVklS8W3bEmVD2nkuEFaNhGQg=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=T0Qg1KutNufk2IM+RhITSkqTqAR85QpK/qRVkMBSYQHLwUxyNed17dolBRIZ86dos TMPBAZy63Nxbt8bzo+cKEztCTXDm3Cr9vDUvPaWjJoCq2lXH0Fg/y3NZswpK84PMMW hlmad5pXPRnoBRBsksZRO3k8cz9HdqFqiorkgIvWks3f2bVb2orqYD0jNTXJEz3zFK GFZNhP1cOo8J4vuyvYew3mBzSU8d4knmbPuyQu6BTHMVU/Nz2axWGtNZZPgfVmDZrx uAgZTfFdZ8vr8+7D8BlkUhnIrSSujQ9yosAqjKuazwT1Jojv7pBgL/IsHTtWP6yKJS tHObw0pgZc5BA== Received: from phl-compute-04.internal (phl-compute-04.internal [10.202.2.44]) by mailfauth.phl.internal (Postfix) with ESMTP id 9D335F40077; Fri, 31 Jul 2026 16:31:01 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-04.internal (MEProxy); Fri, 31 Jul 2026 16:31:01 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdRdBIF8ZH/aPs2g1lRw7GATpB3kPZdHQ9SRq3TCfYELIaqSZraFLrXlLQvfZ+Zw L2FnPfSJd952oWr+y28mIfAsYcvh2g32U0J8iwTV49f0k9Q8hXOe/2BgVhInUEnjhmC2lX 03vYTAlN0bq9NjF03uLbRgPGL4766SVn/XbgM9h+YDdXpmZY+TYoTPsYcygLFMbtetKCiX MKKxau189OoX/tBJGac9kkLntk5lYAb6ORV5iOKO0UE2KpFdBfn5cZVhjAZINCJF+DQAjd IJX9TqJvKbTOKjLKBWundtWMSfFqKbkgkDYizysOgKGZkbbhOs1gMmPcuy0Gf54a13j8K8 mI1S5bYopVWBYIR1xGaeBPsUryTbk45qIhTI0o6U7pv9ryWs4sNkkTY7H7XAuo5/irB2Mi hKevcJm6KEswSYN0jAOu4XPNKtOjhTEn6wMNgsjyRNRwnBQZgKo2gtalJ9j36BiwxGQPhu nIeLdUOLGZHND1yhAJa3vwRzT99ITV1KjmUfaEunbNQug1I0OpHwPvgOyTCO2Tyd72aprq NU7sJhb1jNVK0eFEvzZj8wfsUlUQ4Szc6f4Doha44ZvIyQml/VZ1TT56FFtl64zh9SvvCp tEjZ1reC8QiYxkZl7OjxWRB+Cn71sxvD+/nmwRibTeoH65jmmwPR+VvtfLTg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:00 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Boqun Feng , Andreas Hindborg Subject: [PATCH 15/24] rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers Date: Fri, 31 Jul 2026 13:30:16 -0700 Message-ID: <20260731203031.13679-16-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Boqun Feng spin_lock_irq_disable() and spin_unlock_irq_enable() are inline functions, to use them in Rust, helpers are introduced. This is for interrupt disabling lock abstraction in Rust. Signed-off-by: Boqun Feng Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260302232154.861916-3-lyude@redhat.com Signed-off-by: Boqun Feng --- rust/helpers/spinlock.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 4d13062cf253..d53400c15022 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinl= ock_t *lock) { lockdep_assert_held(lock); } + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ + spin_lock_irq_disable(lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ + spin_unlock_irq_enable(lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ + return spin_trylock_irq_disable(lock); +} --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 70B7C37F00B for ; Fri, 31 Jul 2026 20:31:04 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529865; cv=none; b=XQsFm3QHRU1+kyGhWES1Y2UpP82PFBeJ6aSZ1V3CK5nNKxjcQeUZ4D0N0up9YlD+rh2FYUmLE6HnQTmxmhXGnFFN97jZOzp7w7skzGe1K2E9TbNZ4rBpIR7jxYqWd8lLAerBgSVHKGWL4dDLfMEavTE9PM+cdPDEBgcHLCHRsXY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529865; c=relaxed/simple; bh=SpYxZzYJtgYKNTwiRkmIxqD8f3+jPDEFECfuuq08OzY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=TtpWvhOdKxMMl7juy20898yQUWp7uKnVJdO7GbCn0DvDNkKLZFGvaPR8BxQ1WJugfJurvobMlccBY+w93Sfco/Frk4ParXFD7vtAz7Upx8ABUkeqzBAGYoSDAMIltf3csF6KNShDkCUQxGNqC3kMaA5/9HHOPCS/q9J+r/R0AOA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=cL780KQ2; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="cL780KQ2" Received: by smtp.kernel.org (Postfix) with ESMTPSA id CB95A1F00ADB; Fri, 31 Jul 2026 20:31:03 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529864; bh=0jIqDwRD5BghLbZe9tkUQQ4LGbP7l8lz0sl+eROKvDI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=cL780KQ2yfsWzkw/fi+xTF4SkorGQ8R7mL/YeabnWaXGsT43eyoUQ1JXHbl10YwW/ WchjjP9paO7T0InlGgjCZHaGTHz5hHkb3XM2DXF0Pwn6I/0uF8OnmwLpKqBE/OU+3b WSIAvctvssf5DUuhZ6dolIisKMw4d8MOcJ9xmM3pTmgtD629Agy3mP6Vd0gt/DCbQu Jsc2R/kAh38MJB8qOUjLL20tcQUAQnDqmMbdDogsOGMCq7Rw66ptaXvv9OO26WHRKL eh1Lh24IGB0DjhDr/R6yITydP3XAy45NUULJbs1GaunXvua6qMLlB3fYEdqLOotrJl Zoe7j9p27kttA== Received: from phl-compute-02.internal (phl-compute-02.internal [10.202.2.42]) by mailfauth.phl.internal (Postfix) with ESMTP id 05BAEF4007B; Fri, 31 Jul 2026 16:31:03 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-02.internal (MEProxy); Fri, 31 Jul 2026 16:31:03 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEbBIfeSlo9prBcK5SQKPeqygxB1v/cLTVgZ5l5/ASlZ9vahSgThT0iqkqsNP7FqK 9zw6YF1XG3UNmEM3u6nsEDmLcFtKi/7cfe9xQqiaIubHunNfY02XRK8wlY6K0FXtf9SRo/ 1VhrSN/M5RNZuHROCh24ulCrPMyClee0/fhnnz4fOUOZI3q/RLZ6x47wJ8LNHHXBj1sqg0 b5NEuj6hbqItE5byv8n020XMXAOwtwYIaLnMGB3SEjNqrhWWOtqs6CUvr8EJZH2Eus7heN HM0Lm9s3y2ZTjZXTsYc0ZE4AKwqpvayt6n7d2o32foshM6FX1uxJb77wm/zD1mFUlsqelQ HxbPBLWhYLCHjRoXv9i6XrPX5F1uNRywKOBBe1ZU+G7TXnPfJecbqqVODRW/I8jzx+Smmt jxiESh5oGkeAa5STnrq+2UQUDvf5MTf202FsivzwhAE4lkZhzRA9Qu0LpRcAPd2uJN4ZZj vKiQuiBSKK6YnR16vMQ84qCIn1hX1B85aQQ+wYC30AIvGRjEqrTT1MzK1xk2rtIy86m9Fm COoLpeNymX5vcfTUehte4qAUFhFDu8HJK8PYdRF4N0W+pRcnQnNseXojyDE//KtU8F9juv rWOif2MtYYRQVYjEh4prxw9DL40jY/oJ5PxZxzaOZnsOWIKFLFmAfhCxK40A X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:02 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH 16/24] rust: sync: use super::* in spinlock.rs Date: Fri, 31 Jul 2026 13:30:17 -0700 Message-ID: <20260731203031.13679-17-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul No functional changes. Signed-off-by: Lyude Paul Link: https://patch.msgid.link/20260302232154.861916-4-lyude@redhat.com Signed-off-by: Boqun Feng --- rust/kernel/sync/lock/spinlock.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index ef76fa07ca3a..d75af32218ba 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -3,6 +3,7 @@ //! A kernel spinlock. //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use super::*; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -82,7 +83,7 @@ macro_rules! new_spinlock { /// ``` /// /// [`spinlock_t`]: srctree/include/linux/spinlock.h -pub type SpinLock =3D super::Lock; +pub type SpinLock =3D Lock; =20 /// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend; @@ -91,13 +92,11 @@ macro_rules! new_spinlock { /// /// This is simply a type alias for a [`Guard`] returned from locking a [`= SpinLock`]. It will unlock /// the [`SpinLock`] upon being dropped. -/// -/// [`Guard`]: super::Guard -pub type SpinLockGuard<'a, T> =3D super::Guard<'a, T, SpinLockBackend>; +pub type SpinLockGuard<'a, T> =3D Guard<'a, T, SpinLockBackend>; =20 // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclus= ion. `relock` uses the // default implementation that always calls the same locking method. -unsafe impl super::Backend for SpinLockBackend { +unsafe impl Backend for SpinLockBackend { type State =3D bindings::spinlock_t; type GuardState =3D (); =20 --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 172C5383994 for ; Fri, 31 Jul 2026 20:31:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529868; cv=none; b=d/HTV/sefEiMdTnZhmhCsv2liQKdc+TSfLyP3UWMLiY5V3dvDCOJ++lLrVsWwbbjSAEojSSz4DSu4y6REbdJCy5Oin6J5A+W1JYlCrIP00BLTv+FTRq+IcpSMiiySrytZLIAM9FP8qCIeGcFETBX21UnRpN5wyh0c6l/84ps6JI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529868; c=relaxed/simple; bh=u3Q3W80LnIeTiMoZJbuL/CD4lOmElWYyGl5VGaPpMwE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Ka3m7REGf7e9G43LSPUoooneOp3c2S6Gfynd+I8NLqSSU1S6i8yYbfzgaGlHJ+SZugjA4G6GlmUbuvx6y8NNVF4HDpy5N7VOBnj/Y+LIBYEQAa31eFKsgacybil4JKjP8pPKz7NX48I6f7utXRM4/3jZLDMgGdB8PPELhvRNE2E= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ac+ha5pr; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ac+ha5pr" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 5E7CB1F00ADF; Fri, 31 Jul 2026 20:31:05 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529865; bh=lgAwxtUSmyuA86b+8CVkRASxGnbEa28UJloTXtAnYgU=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ac+ha5przFY49Rm5Qp95FlJe10/hSTOJwkz4/tzEPuPYfzbHPIsfts2H+iQthkJLv +B/rjVDNVCjQW/keK23oTSf+9n4b0tg7YqwfjDsaoqDFooM9Si1YSYn0oBki+QEZD6 JW/fNtI60rAHZ7okWKmfnEbUZosMrmko5VZQ23K2qUwpmBRvt+vKpnpCX5ptbKbgLW jfhbOwAamFB7rCbYkJ88i/hOIIii1/O2Gki+oZca0BZbA2fWDsoKUG+O0K3vwiPx2R SY1fuSloH8hJTSLdi0dVp6wOr2WZFxYaS0mzJ6V7daLckUCFTs0seye4IU9pban8BF KvxzNNfSuYosQ== Received: from phl-compute-07.internal (phl-compute-07.internal [10.202.2.47]) by mailfauth.phl.internal (Postfix) with ESMTP id 8D640F40077; Fri, 31 Jul 2026 16:31:04 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-07.internal (MEProxy); Fri, 31 Jul 2026 16:31:04 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdRdBIF8ZH/aPs2g1lRw7GATpB3kPZdHQ9SRq3TCfYELIaqSZraFLrXlLQvfZ+Zw L2FnPfSJd952oWr+y28mIfAsYcvh2g32U0J8iwTV49f0k9Q8hXOe/2BgVhInUEnjhmC2lX 03vYTAlN0bq9NjF03uLbRgPGL4766SVn/XbgM9h+YDdXpmZY+TYoTPsYcygLFMbtetKCiX MKKxau189OoX/tBJGac9kkLntk5lYAb6ORV5iOKO0UE2KpFdBfn5cZVhjAZINCJF+DQAjd IJX9TqJvKbTOKjLKBWundtWMSfFqKbkgkDYizysOgKGZkbbhOs1gMmPcuy0Gf54a13j8Vy eojwPztPiLBzsgtCODpZ3DGhMoPz3Nk0w+Tw+KmOhYBJ2z4G7K/mUy5EbwtHt9fjJHCbw+ +hBtWMqQKSiQiN668VLT2y/sHgksO7XECwnRHKd0+Q6YvfWY5AlDfUjz34YDyQ0uz5+gyR E07ynZZwPHbeKxVcQyl+3UUg23QY4fLoVp8rPBa/Q2/ANX/MnfXh0zfxGZMfTBl3plsqYN jIIYu43uSlb3/tnME+Hy+QqN8gpnMfpnUVsTuaMat2HXOV7CEqhRyKhEiNruooRGuOTGCk C+sjsD65t0sqPZRh2Fh82CYzjmAB+Vm2yb1/mPkoixIuJgzLBqkG0h8DCijg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:03 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Boqun Feng Subject: [PATCH 17/24] rust: sync: Add SpinLockIrq Date: Fri, 31 Jul 2026 13:30:18 -0700 Message-ID: <20260731203031.13679-18-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul A variant of `SpinLock` that ensures interrupts are disabled in the critical section. `lock()` will ensure that either interrupts are already disabled or disable them. `unlock()` will reverse the respective operation. [Boqun: Port to use spin_lock_irq_disable() and spin_unlock_irq_enable()] Signed-off-by: Lyude Paul Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260302232154.861916-5-lyude@redhat.com Signed-off-by: Boqun Feng --- rust/kernel/sync.rs | 9 +- rust/kernel/sync/lock/global.rs | 3 + rust/kernel/sync/lock/spinlock.rs | 243 ++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..df4f2604ff9b 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -27,7 +27,14 @@ pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBac= kend, GlobalLockedBy}; pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; -pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, + SpinLock, + SpinLockGuard, + SpinLockIrq, + SpinLockIrqGuard, // +}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global= .rs index ec2dd84316fc..ebb10521d8bd 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -306,4 +306,7 @@ macro_rules! global_lock_inner { (backend SpinLock) =3D> { $crate::sync::lock::spinlock::SpinLockBackend }; + (backend SpinLockIrq) =3D> { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index d75af32218ba..6b8f92d5b046 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,6 +4,7 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. use super::*; +use crate::prelude::*; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -143,3 +144,245 @@ unsafe fn assert_is_held(ptr: *mut Self::State) { unsafe { bindings::spin_assert_is_held(ptr) } } } + +/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-= created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on = the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock_irq { + ($inner:expr $(, $name:literal)? $(,)?) =3D> { + $crate::sync::SpinLockIrq::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock= _class!()) + }; +} +pub use new_spinlock_irq; + +/// A variant of `SpinLock` that ensures interrupts are disabled in the cr= itical section. +/// +/// This is a version of [`SpinLock`] that can only be used in contexts wh= ere interrupts for the +/// local CPU are disabled. It can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bind= ings will modify the +/// interrupt state to ensure that local processor interrupts remain dis= abled for at least as long +/// as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`]= token is present and +/// local processor interrupts are already known to be disabled, in whic= h case the local interrupt +/// state will not be touched. This method should be preferred if a [`Lo= calInterruptDisabled`] +/// token is present in the scope. +/// +/// For more info on spinlocks, see [`SpinLock`]. For more information on = interrupts, +/// [see the interrupt module](kernel::interrupt). +/// +/// # Examples +/// +/// The following example shows how to declare, allocate initialise and ac= cess a struct (`Example`) +/// that contains an inner struct (`Inner`) that is protected by a spinloc= k that requires local +/// processor interrupts to be disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// c: SpinLockIrq, +/// #[pin] +/// d: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }), +/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example` +/// let e =3D KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// +/// // Accessing an `Example` from a context where interrupts may not be d= isabled already. +/// let c_guard =3D e.c.lock(); // interrupts are disabled now, +1 interru= pt disable refcount +/// let d_guard =3D e.d.lock(); // no interrupt state change, +1 interrupt= disable refcount +/// +/// assert_eq!(c_guard.a, 0); +/// assert_eq!(c_guard.b, 10); +/// assert_eq!(d_guard.a, 20); +/// assert_eq!(d_guard.b, 30); +/// +/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just = yet, since d_guard is +/// // still in scope. +/// drop(d_guard); // Last interrupt disable reference dropped here, so in= terrupts are re-enabled +/// // now +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with +pub type SpinLockIrq =3D super::Lock; + +/// A kernel `spinlock_t` lock backend that can only be acquired in interr= upt disabled contexts. +pub struct SpinLockIrqBackend; + +/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`= SpinLockIrq`] using +/// [`lock_with()`]. It will unlock the [`SpinLockIrq`] and decrement the = local processor's +/// interrupt disablement refcount upon being dropped. +/// +/// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with +pub type SpinLockIrqGuard<'a, T> =3D Guard<'a, T, SpinLockIrqBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclus= ion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockIrqBackend { + type State =3D bindings::spinlock_t; + type GuardState =3D (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for = writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `p= tr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock_irq_disable(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardStat= e) { + // SAFETY: The safety requirements of this function ensure that `p= tr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock_irq_enable(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initial= ized before use. + let result =3D unsafe { bindings::spin_trylock_irq_disable(ptr) }; + + if result !=3D 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initial= ized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +#[kunit_tests(rust_spinlock_irq_condvar)] +mod tests { + use super::*; + use crate::{ + sync::*, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, + }; + + struct TestState { + value: u32, + waiter_ready: bool, + } + + #[pin_data] + struct Test { + #[pin] + state: SpinLockIrq, + + #[pin] + state_changed: CondVar, + + #[pin] + waiter_state_changed: CondVar, + + #[pin] + wait_work: Work, + } + + impl_has_work! { + impl HasWork for Test { self.wait_work } + } + + impl Test { + pub(crate) fn new() -> Result> { + Arc::try_pin_init( + try_pin_init!( + Self { + state <- new_spinlock_irq!(TestState { + value: 1, + waiter_ready: false + }), + state_changed <- new_condvar!(), + waiter_state_changed <- new_condvar!(), + wait_work <- new_work!("IrqCondvarTest::wait_work") + } + ), + GFP_KERNEL, + ) + } + } + + impl WorkItem for Test { + type Pointer =3D Arc; + + fn run(this: Arc) { + // Wait for the test to be ready to wait for us + let mut state =3D this.state.lock(); + + // Make sure the interrupts actually turned off + // SAFETY: It's always safe to call `lockdep_assert_irqs_disab= led()` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + while !state.waiter_ready { + this.waiter_state_changed.wait(&mut state); + } + + // Deliver the exciting value update our test has been waiting= for + state.value +=3D 1; + this.state_changed.notify_sync(); + } + } + + #[test] + fn spinlock_irq_condvar() -> Result { + let testdata =3D Test::new()?; + + let _ =3D workqueue::system().enqueue(testdata.clone()); + + // Let the updater know when we're ready to wait + let mut state =3D testdata.state.lock(); + state.waiter_ready =3D true; + testdata.waiter_state_changed.notify_sync(); + + // Wait for the exciting value update + testdata.state_changed.wait(&mut state); + assert_eq!(state.value, 2); + Ok(()) + } +} --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D66C238DC51 for ; Fri, 31 Jul 2026 20:31:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529869; cv=none; b=lVmc9Hy6qV9e2KJqreoL6O/x5DsmjyO4VfofBTsneOv7HBohP2myc0h0W5fi2Sp/4X+P/VmtE7Z75nzV46BYdZZezf8ksNgHVNsNTR4DYXxFtRFr9YGJCmUIlMGR3iYJQ7+0nTY9pbriauzqI7rFA3X62Dcd3PUV9KNV7rM373Y= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529869; c=relaxed/simple; bh=i9nGddE3sqAXPLgutnT8ESKgFg7Kb7yokE3ucgABtww=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=M8dk8acqUdSWU3MRSrGvWck3iuCEch+BfmR+ihPYHvjCwqvf7hPT9WAD2zEK65vY2aCk2og0WkVhZQ5HpBeP9YNMX+YcaiDjz77GSmTGZtzr09lBOla+XMz48AIDvJCmwFiUCWxTnn1fRDDkLyYiwyNaq53BQh7YCB4e+HK5akM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=C9tP5t8K; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="C9tP5t8K" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4431D1F00ADB; Fri, 31 Jul 2026 20:31:07 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529867; bh=8YTnDGD6y/NHAmGtt4JsavW6KfpbSRfQwfKSQHrHcMg=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=C9tP5t8KysNhMoEraZvd32X2bdwVx6Id9SO4+CNABts3rAFRLa3cwsW+3l7V2Foof GOGQK4WTtJPp/nOaRIKXefq/G6Zm08muxI2Yg+LpdS/DBcdGJf7BImkX3SNoNCo42e mwy1P1+B6wMXBK2EPTTc1vrauSj6NDPIPE2Gs+7XttuURF1Fs8Wo0iVcF1V1jECK+0 ccXez5XG4E+o2s/Un1bpCQaMXOeHzjVkZTtPM+9sZTuCnP81XWSw55rF9pwJdtiP2Y k0rEl8jFDUHlhFcvIpp24bjqDzcDubhhZ44fesjg4ZQERXURuMErpMJoEdBDWLTz5B CFPB0gmkudamg== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id 73601F40077; Fri, 31 Jul 2026 16:31:06 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-06.internal (MEProxy); Fri, 31 Jul 2026 16:31:06 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFYc6116FG2quGqEuyKIxKnFMEmlYb32kQQ7CtbFGYD9qEYkru11crvm5k1myTjrD mprk+Q3zryg6Sw+VBlHk+Fh2dsP+vkQ+R6SId6D6PQHJ2MNR0r6gfBCLOd3kp8148kZ7F/ KLgIA+5OGrlMzKnGLja63bpFSrAh19/D1KowErF7/fMLoXr6/uBxuTdAfInIovWJuydywf zjiEDrBy2ZQGTG+wDuq3UDJInF4v///JGqWliNx3oB8ndvgTYjGjYM24OlzqaeVHYGQ/UH n/uNgL2CgUXmcpbS+mAJlGjkfOgTDC/OpURLEQyF+W0WAZBtt5IfiFk+NFnrr0x50+Kqpr 2qtvbZBRjsKKzXx2naQYia/F5xoCinX8ZR7Zn3DHW6g1yMMZ66Ub0x3ezqCTCjlnqa0E7I 0dvHN1QgQnNfBIoUtbutNnmbMGAFOSws1xEDWqZ5xW22apxjzKY4NdM2Oyt5wYtYBa7O7w ArcEGVLLCeidiWPDKGpp+B1AReCxPKcjtU3nxjEjCABqveQTuR/CTg0fUU9+N52AFwgW9G xm2E0tLBj4IEtT2qTyslxHccWwHsgikADSt0jpnEdqz0GULVuofg1q/OHPuWCrEFSFhZp0 Y73l2KsixYxlFFT2l+rIbyuEpYlpOubAH+ghzwZXMBv7IaWVcDQGQo0xJ27Q X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:05 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Boqun Feng Subject: [PATCH 18/24] rust: sync: Introduce SpinLockIrq::lock_with() and friends Date: Fri, 31 Jul 2026 13:30:19 -0700 Message-ID: <20260731203031.13679-19-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul `SpinLockIrq` and `SpinLock` use the exact same underlying C structure, with the only real difference being that the former uses the irq_disable() and irq_enable() variants for locking/unlocking. These variants can introduce some minor overhead in contexts where we already know that local processor interrupts are disabled, and as such we want a way to be able to skip modifying processor interrupt state in said contexts in order to avoid some overhead - just like the current C API allows us to do. In order to do this, we add some special functions for SpinLockIrq: lock_with() and try_lock_with(), which allow acquiring the lock without changing the interrupt state - as long as the caller can provide a LocalInterruptDisabled reference to prove that local processor interrupts have been disabled. In some hacked-together benchmarks we ran, most of the time this did actually seem to lead to a noticeable difference in overhead: From an aarch64 VM running on a MacBook M4: lock() when irq is disabled, 100 times cost Delta { nanos: 500 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 292 } lock() when irq is enabled, 100 times cost Delta { nanos: 834 } lock() when irq is disabled, 100 times cost Delta { nanos: 459 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 291 } lock() when irq is enabled, 100 times cost Delta { nanos: 709 } From an x86_64 VM (qemu/kvm) running on a i7-13700H lock() when irq is disabled, 100 times cost Delta { nanos: 1002 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 729 } lock() when irq is enabled, 100 times cost Delta { nanos: 1516 } lock() when irq is disabled, 100 times cost Delta { nanos: 754 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 966 } lock() when irq is enabled, 100 times cost Delta { nanos: 1227 } (note that there were some runs on x86_64 where lock() on irq disabled vs. lock_with() on irq disabled had equivalent benchmarks, but it very much appeared to be a minority of test runs. While it's not clear how this affects real-world workloads yet, let's add this for the time being so we can find out. This makes it so that a `SpinLockIrq` will work like a `SpinLock` if interrupts are disabled. So a function: (&'a SpinLockIrq, &'a InterruptDisabled) -> Guard<'a, .., SpinLockB= ackend> makes sense. Note that due to `Guard` and `InterruptDisabled` having the same lifetime, interrupts cannot be enabled while the Guard exists. Signed-off-by: Lyude Paul Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260302232154.861916-6-lyude@redhat.com Signed-off-by: Boqun Feng --- rust/kernel/sync/lock/spinlock.rs | 81 ++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index 6b8f92d5b046..069fcdb58735 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,7 +4,10 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. use super::*; -use crate::prelude::*; +use crate::{ + interrupt::LocalInterruptDisabled, + prelude::*, // +}; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -224,6 +227,45 @@ macro_rules! new_spinlock_irq { /// # Ok::<(), Error>(()) /// ``` /// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_w= ith()`] in a function +/// which can only be called when local processor interrupts are already d= isabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in n= o-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled= ) { +/// // Because we know interrupts are disabled from interrupt_disable,= we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e =3D KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard =3D local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// /// [`lock()`]: SpinLockIrq::lock /// [`lock_with()`]: SpinLockIrq::lock_with pub type SpinLockIrq =3D super::Lock; @@ -291,6 +333,43 @@ unsafe fn assert_is_held(ptr: *mut Self::State) { } } =20 +impl Lock { + /// Casts the lock as a `Lock`. + #[inline] + fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisa= bled) -> &'a SpinLock { + // SAFETY: + // - `Lock` and `Lock` = both have identical data + // layouts. + // - As long as local interrupts are disabled (which is proven to = be true by _context), it + // is safe to treat a lock with SpinLockIrqBackend as a SpinLock= Backend lock. + unsafe { core::mem::transmute(self) } + } + + /// Acquires the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock= ::lock()`] function when + /// possible for [`SpinLockIrq`] locks. + #[inline] + pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) ->= SpinLockGuard<'a, T> { + self.as_lock_in_interrupt(context).lock() + } + + /// Tries to acquire the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock= ::try_lock()`] function + /// when possible for [`SpinLockIrq`] locks. + /// + /// Returns a guard that can be used to access the data protected by t= he lock if successful. + #[must_use =3D "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock_with<'a>( + &'a self, + context: &'a LocalInterruptDisabled, + ) -> Option> { + self.as_lock_in_interrupt(context).try_lock() + } +} + #[kunit_tests(rust_spinlock_irq_condvar)] mod tests { use super::*; --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 21E1A38E8D3 for ; Fri, 31 Jul 2026 20:31:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529870; cv=none; b=ibh3JFM3FrBeP5HXZBnge5uKfJtzPif057X3b4Lif06NsHFX6GMAxsfPCNy3v+fN3tlOHs52BUclT209QBvC33elRHBjCtGWkOO/THWMRZBi3DMXv7JU5u7BU09fTO6JKaY7/6KkbsX4AJWdReSBrqm5JSahIBajJz5BZLqM0VI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529870; c=relaxed/simple; bh=l03Un0caKINAvj1iVlE/sOliFO+j6NY979CM/DpkRIA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=O/2udv/469dHnNnw9PlJn6y7iPTjlcue4RmQvhTd+SkzL7ppXJ55AdmXIlAV0uNcAaHFJs3zuFRdYc5VfwQUGvU90bNFUKaaluVfxNaN9DQZ/OG0+gmguZvuab3GOSrn8R+y9wJA/NdqbJGxKOd/SEzsldL+DG+67leLWfJJ6Ns= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=DJbjHWpj; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="DJbjHWpj" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A9A251F00ACF; Fri, 31 Jul 2026 20:31:08 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529869; bh=WesjdpTDaGBH3ilY8RyXNXpARBP2O1BbBYtfR/J5ZFQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=DJbjHWpjef8xKnQx2e6qzhCWW8YiQTgkLRwerQBWsP8eEmiTpEqBfo8MnNZTj4CVl SfxWuApV5SipfSX05XtmoEAJjkalpLtoHCPjSR153zIbU+lNZYUVgl0ImzvxHm6rcZ c/D8hnxmLnoup6/LOoCrzjxreCIbQLii878cLUpCoCkarM/jw/V3fnX1tixjYzzdZY wTzRILCj2W7wSF5O7JYwQwSpnOjmFMe4VWxS7dEX+Yh2LHlZ6kNh1bsfABEm4RYTVO FfWiLIyFPc3PMwweVP8paJwC1gR3+5rJTNUZWbx989/P3H/4JrH3gwNO79F2rJQvKr SHSVBvy7jZT1w== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id D8D4AF4007B; Fri, 31 Jul 2026 16:31:07 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-06.internal (MEProxy); Fri, 31 Jul 2026 16:31:07 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGcyIDgVOSf66q7sFNbNg6dhONC3pWEknHmR0ZDERyZUv02pF47PwtWMGHF/GCBjs z8BQ3yI+0zTbDVWXZCeuTrRzOL8fR/milpuDWnN1rI1rwcGVW4BxbmsCmf2KzA1TqOz0wR bejBL0O9zByiVv82ITvuC1hg+yxRC4wfS/0M6cf2PIQvqrx6+cDFmfbR2mt1VblliMIbG0 nfi6RHbRCNvvdj5grRsIArMGlRRAahBOU3g8lasFtoOMQxo3Y110eVl0Gppq5OUcHq7NMN IPfg7+7sDGuhtw3rxYX1WQjsOYaZ4suTKpOwpQO5Cg5VyIkond9V3yhPCyyK5zuX7QbxXe aD6snuB5k5zwL7LcyMQRBuyE2Hg4ZyFnoeCTOe8WAslocaH1LJI7rg8Za8d9rc34FOvwAo eiJzOopW+JLSOSq4uvr6j0fI5PIwSMcrB+CVfZRANtXNFaVNRs3eSeiel9tr9zUUKzzW5n 7R9mcJADXp3iV72H2ZozUoS1eVp6NcKlpZ7kvBDvFaWk9gUAfxE5lZh5rQGJl6pkB9zvvc Qbnk0aCVpF+z/nnn0JG9WVALfAcApyIF5qvQ1IMBTflUdx11Zpz4payIr1VY7Nz21Hy4dW j/9u9jYhkGJBKClFRBCE1hJyuaTI9BX8wHNRmfRrCgTK/gVxLo5voq9pGxTw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:07 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Philipp Stanner Subject: [PATCH 19/24] rust: sync: Add abstraction for synchronize_rcu() Date: Fri, 31 Jul 2026 13:30:20 -0700 Message-ID: <20260731203031.13679-20-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Philipp Stanner synchronize_rcu() is a frequently used C function which is always safe to be called. Add a safe abstraction for synchronize_rcu(). Signed-off-by: Philipp Stanner Reviewed-by: Onur =C3=96zkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo [boqun: Fix rustdoc reported by kernel test robot ] Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-3-phasta@kernel.org --- rust/kernel/sync/rcu.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs index a32bef6e490b..d867240be736 100644 --- a/rust/kernel/sync/rcu.rs +++ b/rust/kernel/sync/rcu.rs @@ -50,3 +50,19 @@ fn drop(&mut self) { pub fn read_lock() -> Guard { Guard::new() } + +/// Wait for one RCU grace period. +/// +/// Waits for all RCU read-side critical sections (such as those establish= ed by +/// a [`Guard`]) at the moment of the function call to finish. +/// +/// Does not prevent new read-side critical sections from starting, which = may +/// begin and run while this call is blocking. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn synchronize_rcu() { + // SAFETY: `synchronize_rcu()` is always safe to be called from proces= s context. + unsafe { bindings::synchronize_rcu() }; +} --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E014B390986 for ; Fri, 31 Jul 2026 20:31:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529872; cv=none; b=fnC5K5KJ9THvYq/FIcJrUYb0/atpb2VpgDY8s+P+Q6zA4OaY1BJB/wUmA00FHF2LwMRMnrOHv+4Ro82OVTfQK+PrzI1750af81P3aBviaGkVFLKUFnj2N8qybCAtEdble2X7pD5OABdBHwUbaiJpXswam6RMvLgNXJt7DiXRBFI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529872; c=relaxed/simple; bh=2f7f3gbtPo/Uq28ZOW5NymCCIuSaR6PvKX7O22oZJ9w=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=sifuEDXLjGX2bm1w2NMHrTeArN5mFtlfki5j/mRnPPrGCRD0slpnHJGWeE7dtQN+ek1k7Ol+iQu9RR+2evg5DQw0tLhywvqPSJDtOHUadjTbpkibHHT9iXW4Qpwbdnn/zmSp8gH3l8+3Udl+Lp3jw600TV1taZwHdJhU/HNPyd4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ma3yYNF8; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ma3yYNF8" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 2BF131F00ADE; Fri, 31 Jul 2026 20:31:10 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529870; bh=MsIDZiwSPmskK9n5nhtTssL2Y1o3vOS3ZV20vncjul0=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ma3yYNF8izHoiKeiPE/lFK8PynI/lAU9XNRinAfqcoUzj9O0LCQ101kkcqJEsgwdf wavpDCiVp0m8MiPjixK/3kH8Uq0bEpJdYAh4G8FdmFXjgUFFRB2MGn+c6y6xp9dMLs kOXkJoYxve04Ia2898cAG9k7jeITyctCLO9mAZDTrfZJmdUGEUeoFTvwDhVNeAC+35 gbbpEeZar9J7g2aNkehRWSoH2To0nZlQ+pXoQJURdvn761BsCCR8SvsbNrV3njVJZQ 5G52GJcrMJKf27ttsFF5Nu/A7DJYEL8M3qKRd3x9y4tR0CDFHzmenphHuj1kAcoQxT 6/sJHA0zFJTyw== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id 58F68F4007B; Fri, 31 Jul 2026 16:31:09 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-06.internal (MEProxy); Fri, 31 Jul 2026 16:31:09 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEx/5S6G4+/qWahub76rD+/ZVHwV35J6FmUal1s+rCkFqtdmVSwk6K0IcMoYNBriQ jzkD/QIWyBCXxFXrUxewwooRRYY1snGImAM8ms1J0rQ4yousOqpNJ30B7CHK//nNcomy4W I6ocsiYNJ0mCNaJmOgh+Pf7hXCJ/QEJRpl7PHOpTLOD8FaU/mgJ38GevRTx88Y8aOyNMPA RxPFyEJNVBAYxKhXvQwp05QVk4kSq+dG3VxPm0kHYbTYKcWf4HJeBmpgQRwXCF1tnzxJ3p u5VrXdml3B2K+pxowi0aeuUo4V1i7eGjDAPpMqKgE9ULEmEwsndt/ZtUBMuM3XtcdFbydq cozzsDoeyagIZHo14etb/oYSZt+8QWLZ7vnFKvZkxuFb8hDYpkszdxNLKkUQFE5Q+wCkoA NPPhEQ4VvwBEIb7M6Uh/FXp8LvcPyre2JffvbgObG6y/e0lhAd3YxuPs3/Al+j4fy0vTCP o1Vt3U7VAF156MFqK3yDvg1m80eIJ60n5bImutblaWA9uO2qC4yu55Og0GaylvElS6TGQv XtpvGS3bSwgSJul1qk1tY4OLmFI3k8z+unbequ5U8jUCbl88j6ZdgSCzPQTkMpcDnPl/ay v3DnwzcBnDQH9Isee1CU22DUuUYpGb92efjZLD4sDFIYVZsSzhSlOjSQXwQw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:08 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Philipp Stanner Subject: [PATCH 20/24] rust: revocable: Use safe synchronize_rcu() abstraction Date: Fri, 31 Jul 2026 13:30:21 -0700 Message-ID: <20260731203031.13679-21-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Philipp Stanner We now have a safe wrapper for the foreign function synchronize_rcu(). Use it in revocable.rs. Signed-off-by: Philipp Stanner Reviewed-by: Onur =C3=96zkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-4-phasta@kernel.org --- rust/kernel/revocable.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index 0f4ae673256d..f539603349f1 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -7,7 +7,11 @@ =20 use pin_init::Wrapper; =20 -use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; +use crate::{ + prelude::*, + sync::rcu, + types::Opaque, // +}; use core::{ marker::PhantomData, ops::Deref, @@ -161,8 +165,7 @@ unsafe fn revoke_internal(&self) -> b= ool { =20 if revoke { if SYNC { - // SAFETY: Just an FFI call, there are no further requirem= ents. - unsafe { bindings::synchronize_rcu() }; + rcu::synchronize_rcu(); } =20 // SAFETY: We know `self.data` is valid because only one CPU c= an succeed the --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CF04E3911AF for ; Fri, 31 Jul 2026 20:31:11 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529873; cv=none; b=W+ytntNHigjxNdjbL5k67Zed/3LhgiHaEfio2sycCpcrg/mkGjyITkhrKccGGDsPHKs40i0TX1RvcYSsAcGpakVh3ryX8KX/GSdcb3X3uYl/fJUcf6FkNtGscDcG+DjAv9SeK/MLnxj35ZoBbOEYkoxv72WyGZUzqwaea/p/65I= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529873; c=relaxed/simple; bh=/R9Xt2cq7wR9LFr7KE/IiwroZVU+4G7lhcrzcWPSLzo=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=aAcYzImvkrGYnv+7TgtZzfP+Jh2zZ+VQxHXCcD7jjnIIR2lIHLg4CwJ2Xd3bCKORxg/bi9U1HwzFO1mAEMimgJhJRW+/z3vEGrrBCHxSK1f/TeYIw0LFA2Ivqbn/T23EBIFpeor/GhQblIJ3WG2Ry6qKN7G/WubKWTxQ09xBcGo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=H8ne9B/9; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="H8ne9B/9" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 99CEE1F00AC4 for ; Fri, 31 Jul 2026 20:31:11 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529871; bh=yS8RNIIgYjKxdhvNp8TorwPekD/QFVeixdVx0VMVBqk=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=H8ne9B/9osMdxyRQ+2V4tMcDGNQjG8kiAwNnQjVo0M3ERyEIDJ04Ze9U0ZPqOZK2D Ctjk+cohEDxRpvPRVRDc2mtvcb37LvHyQ0maceiZkdmX4vcVbw/RMjM/5xmqXRLddQ Mv9QGSFrD8B5XqrkJbtYepKIZ1WMS0/eljcWI+OTFbU+IvYgeVPa/Y4Q8Q2K54PwAn vg1qa4cRHpSF7h+RvW8ycT7T+l/RZ0ROaBXv1+e9vsxPLBhLEPFxKksmD7McV+CHsF DkJa2teE0GZvDVRC8JGPzgall0p/03Ix8hZVl/6fKJdc5ln3cwn6LpeDvbs2yVydj3 dNQUlGLIuOiLQ== Received: from phl-compute-03.internal (phl-compute-03.internal [10.202.2.43]) by mailfauth.phl.internal (Postfix) with ESMTP id C78AFF4007C; Fri, 31 Jul 2026 16:31:10 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-03.internal (MEProxy); Fri, 31 Jul 2026 16:31:10 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTF86Va7Dqpiv0HRB/Rt3CM1ZfwUI8sFdbNEysl2vjWzKGQHPL0vtJNxMWzwq4RCT7 5MXz8yAQaenZ+l6ReNiQBMk90me+gM63MoAJ/OG6881cu1guy8MfcdgMgPRXqCdnOwY3UU Yre4T7XNVpX+hYLj1cR+5sDPDlXBwMgddj0S3Pt9EBEeMQP2C2QxRT89euxxh1RusxuOUE pjdPkRkdvSWh4KDKL7SA/RMgPi4/GzhXt7G4+N2ySbTjJeyTz6zE/2g6iZFhSAe8C+XsYy rTsuNdtrWKoEzpDq2SUPcIxKOwDbQW+j2sU4NoAonSXuB6sJUYCYCWtPGQc2h06l1wGha3 VbqFlrpaSvkUv4cSFs/CIJZekmAwTH7l3kfd4bn136EBs1v7idVNEtI9xOp4t/6dYFYoZj HNOiwWO7BCYFcV86/PgAej8VCrrAn4x35QwnZ5IqN4BBMr28qVpGCj6kJN1xx4dwTLLt7l kFyGCjoNvm+ex0wlejYEREQe+6JPMXlzrlNlFaNFyE5mQoTWk3m4OxcpN0Oag2DIM720KN jP/YnkStKA4tkGWd+eCWSdlduwDEMcYWRoClRGFG2purHU6z0ts5HIiGTGxaiKe4iJY6lR megd0Ra+E6E9h6NTlhpvMgCOjoQcrVUbWV27mPjlI+HhP00RxJSjNfa7zK2w X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:10 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Philipp Stanner Subject: [PATCH 21/24] rust: sync: Use safe synchronize_rcu() abstraction in poll Date: Fri, 31 Jul 2026 13:30:22 -0700 Message-ID: <20260731203031.13679-22-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Philipp Stanner We now have a safe wrapper for the foreign function synchronize_rcu(). Use it in poll.rs. Signed-off-by: Philipp Stanner Reviewed-by: Alice Ryhl Reviewed-by: Onur =C3=96zkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-5-phasta@kernel.org --- rust/kernel/sync/poll.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..5aa0ce9ba01b 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -8,7 +8,11 @@ bindings, fs::File, prelude::*, - sync::{CondVar, LockClassKey}, + sync::{ + rcu::synchronize_rcu, + CondVar, + LockClassKey, // + }, // }; use core::{marker::PhantomData, ops::Deref}; =20 @@ -99,8 +103,6 @@ fn drop(self: Pin<&mut Self>) { unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.g= et()) }; =20 // Wait for epoll items to be properly removed. - // - // SAFETY: Just an FFI call. - unsafe { bindings::synchronize_rcu() }; + synchronize_rcu(); } } --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 054D23806D8; Fri, 31 Jul 2026 20:31:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529875; cv=none; b=bCvUHVCBGkRNnKGA43ycqKGjvsvEuvdxkCH5qoyOrQV3ZZWC4Rs6UbFwV6OJ+wGvwovTDoSKFu/I4oE/BbD9G1DzWf6dnIJTIESPgC24NSswG/PsvV5ZDa8F0mTXr4PcXeClRdDZ/EyPW+gdHs1zlzJWwGpKH2aedX/89ZGeeyc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529875; c=relaxed/simple; bh=g7dAukeoDvfLBNcg9vE5PsrRy89rj838ZPJP9rjLfKE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Z4vwSnAMFPnM7nkj8raKwcgWWgq+VOwMwaYqYEfxR41GCMcHbODY2Aox7fk76O7h6XaCUzn+IJYo4kQ5QDPRxP5LLSEVKDtXaiHqCg+6VTNuK/Ip02CQeogjgyreBaGmuYkmU/r6VZsRXmTOMMGBU3A7WJTyKeAiSeSjOizukgQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=F5VJVBfk; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="F5VJVBfk" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 2E8C31F00ACF; Fri, 31 Jul 2026 20:31:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529873; bh=4pwt7mRy78FKv5uQ/1ds1AYEuqBc5vFVkMphAXXIIeA=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=F5VJVBfk6EHmMO4O0Fz4asw+X6uK4VSta4XKhjLSW+J7ciePHOBHddptNb3GN0iDR IS7swtxm5ck/q0dA5yWUULRJAB1589VuXbTC9jgR5Upg1PRYTIiwsrXUYteu3mzSvE 8NtE0WU7TIKh6R7fDsWq6khWgbJYcKwr0uayUm/tUhGSX2WhWwRunLdTKCrEoQfllC k0OfcH2VP1UpbmDl1xYGOO35idex0jzEDjpboLuXtHQaP1KxJ9DJYv807OWOUHWnhn jw6Sa3vsdefxr+4zx0n2bRjg0C2jE3qXxojT+tGdMeMG82xjJEEY1UQ+OLo+/fHLLk w8XstG+hbx8Bg== Received: from phl-compute-08.internal (phl-compute-08.internal [10.202.2.48]) by mailfauth.phl.internal (Postfix) with ESMTP id 5B6F7F4007B; Fri, 31 Jul 2026 16:31:12 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-08.internal (MEProxy); Fri, 31 Jul 2026 16:31:12 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFps7Fcl2lDQN8QuKjkQ2Twray9XFETCWlSba0KMthHKuWpzT2QuJsxYlE+Jfwxxk r4LumXM/JVSIx3esVw1aDMeAJaVHLr+c+QoTtbKWA/cjowqeMJQoXkNExL9SUShkGISMJN I4Jxy+s/E1/lkenAlXh9Q30vQuf9rA/PbrkWD4pKxMAkC+U6f3km0ithIr0S+TXfbV+4FM rMsII+vjt3nbbUrqTbYDv01BF1FGDJ2EkkIzsv92rcLPomBtS/DYv0HD/S/dv1En38CiLr u6+5RUMJmyRxjeZZwQND0Q2Cx0WP3XHFSY1h6xZNSfNwcAc/Ongt++m0x7heb8YD7IrwXo YkIhG6KuZJPuVeQrNTB2UMxd1Uoq8IIA9r4H/UGvxMjS3M4ZE7BaK5Z1k2Q3zSvQVJ928j m1Na/eOtEqKoCbXz0RR7pxMaablV2LIs8dyV5GXjrRFey8/c14lDSARIJu+/MVuBcf6Gkf 93/FCbLPkncFFoAkpdCJeQj648mUu++7TauqMQyPF/2t62QiyPGRZ1fIqEML7KpzKjaSGC HewsjQNPVMQBDjNkrbsqDt7Cokh39zo65s6QNxjCFEAO5v/rLi5N4YhcoVLt7KaF9zLbmd 0O3uWWQcATkYYyl/C12E9w7QS6qF9epWYq7z8B/AIfduXheV9gy/sojp6CMg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:11 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Eliot Courtney Subject: [PATCH 22/24] rust: sync: Add helpers for mb, dma_mb and friends Date: Fri, 31 Jul 2026 13:30:23 -0700 Message-ID: <20260731203031.13679-23-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Gary Guo They supplement the existing smp_mb, smp_rmb and smp_wmb. Reviewed-by: Eliot Courtney Signed-off-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260609-rust-barrier-v2-1-30fcc48e1cd0@gary= guo.net --- rust/helpers/barrier.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index fed8853745c8..dbc7a3017c78 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -2,6 +2,36 @@ =20 #include =20 +__rust_helper void rust_helper_mb(void) +{ + mb(); +} + +__rust_helper void rust_helper_rmb(void) +{ + rmb(); +} + +__rust_helper void rust_helper_wmb(void) +{ + wmb(); +} + +__rust_helper void rust_helper_dma_mb(void) +{ + dma_mb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + __rust_helper void rust_helper_smp_mb(void) { smp_mb(); --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 09AC8392C56 for ; Fri, 31 Jul 2026 20:31:15 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529876; cv=none; b=rptrd/AwcmSSeLAbM8dwifJMrtMikEx7XR6xCX7r/6ZCydWTSqJzlSsknv3P8Uqx/eRNbGyAxzSQwEEN1JYWET67vWkGMoVjfNpU52K6x17q7UTnKh2e7fp7QechU6i4F2eja0Pv9D6CFsfyoAHHhtHr26zr5qwGlnrJyRYwjpI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529876; c=relaxed/simple; bh=BnZY+ssx1FNk1UN/oxDrG+BGtBCN1VUyUUnZuJBcyTI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=PDkhvk24T8ZwDr9MnWMlXSeTdBQUcY9AaH4/Zl/osFTlxSGZDHTu86cDnhtv8uPJ2x3sCLo8y7UJRek9XooWY6FrfY1Npa/UC+tOJ9n2QwDhcMvK5h7KOu2OLKGBV2V0XVYz6yqP9zpvapOXiCMuqcqWd9Ns4feUTYpe+e/B4mg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ZaFNHuwT; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ZaFNHuwT" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A5D301F00ACA; Fri, 31 Jul 2026 20:31:14 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529874; bh=ee35tIBuiy+GnxuKIVMP3Asj7TkW4wEaj81yaDPH2Yw=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ZaFNHuwTc9/FrKvOQTkvztLFrgVfei98TxpZ+bE81TuFNcZjzXprQvm4uL5bk59Lq DsRQwJsStF4e7jOECqmBwPxfs+Sj/YAZycKgt7G/IymnXpi3ZrDuw+v0nypw/TntEn IbbETamzJNf2hGRWxvStMsord9yZRhl84TL8hMBL3RSozI5IQbRyb2BNW3Cr7hLbmj TXDGsFLOvLkDSSJc7NNJsTExO8E3kxLmMRaLq5/iRC1Ntc4hVj/ueuvI9rzdpfA0ny vt1e3dAlCUIAb/XpDSg9iGRzbrTyb9DaLuvY654v/LhimmLJ5rSd7ch6Nea9NF5ck8 t8Q3naiaXFeGg== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id D4F6CF4007B; Fri, 31 Jul 2026 16:31:13 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-06.internal (MEProxy); Fri, 31 Jul 2026 16:31:13 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEbBIfeSlo9prBcK5SQKPeqygxB1v/cLTVgZ5l5/ASlZ9vahSgThT0iqkqsNP7FqK 9zw6YF1XG3UNmEM3u6nsEDmLcFtKi/7cfe9xQqiaIubHunNfY02XRK8wlY6K0FXtf9SRo/ 1VhrSN/M5RNZuHROCh24ulCrPMyClee0/fhnnz4fOUOZI3q/RLZ6x47wJ8LNHHXBj1sqg0 b5NEuj6hbqItE5byv8n020XMXAOwtwYIaLnMGB3SEjNqrhWWOtqs6CUvr8EJZH2Eus7heN HM0Lm9s3y2ZTjZXTsYc0ZE4AKwqpvayt6n7d2o32foshM6FX1uxJb77wm/zD1mFUlsqeFO BH1yQ7M16W+COuLmbkYrpywWYFxLYn1MsVfNSkehl1kbBbS5aSCV9DZODkwyx8HBrq2M2T OwZO5o6kx3UD1/PYVo4ZHjY92hv7+0X0VsPIslkiBQ/QboJ0GRYyH9lx/1OI694XfOu2aK bSwVQVwlVWEStRbQb7H+uGtFB25JCAViVYAMW1ArzEilw3rzBhdbpbl9LBIPm6EuANrdS2 lPdYSfEg5cub4rVa+KOnWQsZpPqLlG53vfzxx9q4ZccNnPXc6/Z1hJA/itJFNilqJ69ohK cLguu45uKprn3Jytr5v1tnbiq7aNnSaJtgjh8YH/ojjMPJmN2JS7OdwWcOCg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:13 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH 23/24] rust: sync: Add generic memory barriers Date: Fri, 31 Jul 2026 13:30:24 -0700 Message-ID: <20260731203031.13679-24-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Gary Guo Implement a generic interface for memory barriers (full system/DMA/SMP). The interface uses a parameter to force user to specify their intent with barriers. Provide `Read`, `Write`, `Full` orderings which map to the existing `rmb()`, `wmb()` and `mb()`. Generic is used here instead of providing individual standalone functions to reduce code duplication; for example, the `CONFIG_SMP` check in `smp_mb` is uniformly implemented for all SMP barriers. This could extend to `virt_mb`'s if they're introduced in the future. It would also make it easier if new ordering types are introduced in the future (e.g. `Acquire`, `Release`). Signed-off-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260609-rust-barrier-v2-2-30fcc48e1cd0@gary= guo.net --- rust/kernel/sync/atomic/ordering.rs | 2 +- rust/kernel/sync/barrier.rs | 127 ++++++++++++++++++++++------ 2 files changed, 100 insertions(+), 29 deletions(-) diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/= ordering.rs index 3f103aa8db99..c4e732e7212f 100644 --- a/rust/kernel/sync/atomic/ordering.rs +++ b/rust/kernel/sync/atomic/ordering.rs @@ -15,7 +15,7 @@ //! - It provides ordering between the annotated operation and all the f= ollowing memory accesses. //! - It provides ordering between all the preceding memory accesses and= all the following memory //! accesses. -//! - All the orderings are the same strength as a full memory barrier (= i.e. `smp_mb()`). +//! - All the orderings are the same strength as a full memory barrier (= i.e. `smp_mb(Full)`). //! - [`Relaxed`] provides no ordering except the dependency orderings. De= pendency orderings are //! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. //! diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd94..1180695d533a 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -7,6 +7,38 @@ //! //! [`LKMM`]: srctree/tools/memory-model/ =20 +#![expect(private_bounds, reason =3D "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and = rules. +/// +/// - [`Read`] provides ordering between preceding load operations and suc= ceeding load operations. +/// - [`Write`] provides ordering between preceding store operations and s= ucceeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses= and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{ + Full, + Read, + Write, // +}; + +struct Smp; +struct Dma; + /// A compiler barrier. /// /// A barrier that prevents compiler from reordering memory accesses acros= s the barrier. @@ -19,43 +51,82 @@ pub(crate) fn barrier() { unsafe { core::arch::asm!("") }; } =20 -/// A full memory barrier. +trait MemoryBarrier { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) =3D> { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. /// /// A barrier that prevents compiler and CPU from reordering memory access= es across the barrier. -#[inline(always)] -pub fn smp_mb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_mb()` is safe to call. - unsafe { bindings::smp_mb() }; - } else { - barrier(); - } +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias =3D "rmb")] +#[doc(alias =3D "wmb")] +pub fn mb(_: T) { + T::run() } =20 -/// A write-write memory barrier. +/// Memory barrier between CPUs. /// -/// A barrier that prevents compiler and CPU from reordering memory write = accesses across the -/// barrier. -#[inline(always)] -pub fn smp_wmb() { +/// A barrier that prevents compiler and CPU from reordering memory access= es across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devic= es. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias =3D "smp_rmb")] +#[doc(alias =3D "smp_wmb")] +pub fn smp_mb>(_: T) { if cfg!(CONFIG_SMP) { - // SAFETY: `smp_wmb()` is safe to call. - unsafe { bindings::smp_wmb() }; + T::run() } else { - barrier(); + barrier() } } =20 -/// A read-read memory barrier. +/// Memory barrier between local CPU and bus-mastering devices. /// -/// A barrier that prevents compiler and CPU from reordering memory read a= ccesses across the -/// barrier. -#[inline(always)] -pub fn smp_rmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_rmb()` is safe to call. - unsafe { bindings::smp_rmb() }; - } else { - barrier(); - } +/// A barrier that prevents compiler and CPU from reordering memory access= es across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias =3D "dma_rmb")] +#[doc(alias =3D "dma_wmb")] +pub fn dma_mb>(_: T) { + T::run() } --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 12:19:52 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B492A3955DB; Fri, 31 Jul 2026 20:31:16 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529877; cv=none; b=EZs4fG4B2cxBpjrVbz+aEWSN6Bzzid7QffZKDFAlBt4Jp5VLSt3gSIoefK1ud4G1fgQSdJpTKFE3ufrD3bidiCYPAYIz5p41Vv/1DxFBmxUYUjxmKLZ7iTQeLXIpY9C474ylQnoyPQZ+dYE96cU1uzKMVkpUvh6ruJQsGRwfZ1Q= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785529877; c=relaxed/simple; bh=GrWQGOL3crZsuH45+ZBPVTITQEqIF9Q5mjZVqqOZsZ4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=dgQqB52TgPvyxlmgYJdG94247ZYTRjt3ZfEQmdtqe5p6YE/cHo0Q0DXJNkh8DZmS+APrC2gs83E2BX0rGAE4q7u7r3907VIxn64jFS0UkdpQm2Y7ABaVn8CxHcf50YhRWelUzLsYaa/baz6LkYLhvb6sLn4xvYd4zMybluv/pzk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=TxeSpZ49; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="TxeSpZ49" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 219331F00ADE; Fri, 31 Jul 2026 20:31:16 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785529876; bh=wVEFpBGWMoM5WsmfsEJRp4HAJRBtzzCQMV8wkIWA9rQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=TxeSpZ496vh3Yo2pbWR2XzTzeumr3yggH4j+tEVrgn7nH7EMhP59obk3U2ugx/4L/ zBU4D4GK0KneF54vFbmC5lYIgP5M1R2uc3dnnKP2KJr6Mm1UT1ZB5Dqkzzc+Is0I46 26sU6QCMBLsmbxfDYT/TyM3C7/oKvHThkkZXm6/HbME/6ad3Zlr9w4D42GiGCVT/UD UnLi0MHmbAyqfH6V448qcbxI8y+5/HJ+UmJb1gtMx8Y04uocPp8n8l7FZK/845RaS1 1jGB1SDPcipfucUDya5i8KosOPERCdvO5INxi6ugGL1SwOU60cIWV31s1BEQ4/guMo 7ePRBofUanESg== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id 4E179F4007C; Fri, 31 Jul 2026 16:31:15 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-06.internal (MEProxy); Fri, 31 Jul 2026 16:31:15 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGOnHZUa8WnHr5e/dCZtTVFW9LuLZK8c+qqG3l4XfTIbozZf3AA8IlPWbaPJrJVqp nWy7ITIOPTqUi2f0a/VJdkfYNjO5w0jTzFUK85w5FITvZQKodwqJRThBmnyHC0Cie3wi69 ITHmm+/NEbE/QfmfHsFZ19G2bqZRRrdc6xFMB8o7R63Vm8sJNcJhvF8FFEQ6zjWiUuCXdq //OXBuOYSvSsZdaeP/DKCPX5DV8wK62MR2VRd+FJc5tN/aWsAyfghCYhqta/P/9bXeKNSG ky2jVCp1qqxDCiPr6HsaFpF7IpKVkoIE+I2nA6kdnfuEbvPsH/252rhfRbSUdXx1DzxxPO KnpWTMajU3wteTOHQfea4t7kv8pOleUOzG3Pt/hvrX+GXSpnVzG2ZQFyHA1tigzL5dgAPd tZviqdHlfFc1UHGe9h4hQCY6bAdcD1/K3C06F1inGTrlsI4rP85Qx0TonGpliuSui9KkZY rf/Mb7pgwWw2ggmq2m/IiIQLDdCYOJEC2R7uXoEbfRQkPw1emTQYoytvflBxTYMiuGIeHA JJuZ6Rw+vjscmAYe8F1RtjgS0rav7Hn4CK/bGLfcTNe27ryS3WfzSDT6UohGZwh4p/+rid IUuL8z4J6v7o2Jd+AlmVk2SPiqjrv34hRkgHWrZ4H7wwnQKrR0stj33Wupzg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 31 Jul 2026 16:31:14 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, FUJITA Tomonori Subject: [PATCH 24/24] rust: revocable: Use LKMM atomics instead of Rust atomics Date: Fri, 31 Jul 2026 13:30:25 -0700 Message-ID: <20260731203031.13679-25-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260731203031.13679-1-boqun@kernel.org> References: <20260731203031.13679-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Gary Guo Kernel code should use LKMM atomics. The existing code is `AtomicBool` with the need to use `xchg`, so convert it to `AtomicFlag`. Signed-off-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260716145536.3681630-1-gary@kernel.org --- rust/kernel/revocable.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index f539603349f1..0e55e2a0fb37 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -9,14 +9,19 @@ =20 use crate::{ prelude::*, - sync::rcu, + sync::{ + atomic::{ + AtomicFlag, + Relaxed, // + }, + rcu, // + }, types::Opaque, // }; use core::{ marker::PhantomData, ops::Deref, - ptr::drop_in_place, - sync::atomic::{AtomicBool, Ordering}, + ptr::drop_in_place, // }; =20 /// An object that can become inaccessible at runtime. @@ -69,7 +74,7 @@ /// ``` #[pin_data(PinnedDrop)] pub struct Revocable { - is_available: AtomicBool, + is_available: AtomicFlag, #[pin] data: Opaque, } @@ -88,7 +93,7 @@ impl Revocable { /// Creates a new revocable instance of the given data. pub fn new(data: impl PinInit) -> impl PinInit { try_pin_init!(Self { - is_available: AtomicBool::new(true), + is_available: AtomicFlag::new(true), data <- Opaque::pin_init(data), }? E) } @@ -102,7 +107,7 @@ pub fn new(data: impl PinInit) -> impl PinInit= { /// because another CPU may be waiting to complete the revocation of t= his object. pub fn try_access(&self) -> Option> { let guard =3D rcu::read_lock(); - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // Since `self.is_available` is true, data is initialised and = has to remain valid // because the RCU read side lock prevents it from being dropp= ed. Some(RevocableGuard::new(self.data.get(), guard)) @@ -120,7 +125,7 @@ pub fn try_access(&self) -> Option> { /// allowed to sleep because another CPU may be waiting to complete th= e revocation of this /// object. pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> = Option<&'a T> { - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // SAFETY: Since `self.is_available` is true, data is initiali= sed and has to remain // valid because the RCU read side lock prevents it from being= dropped. Some(unsafe { &*self.data.get() }) @@ -161,7 +166,7 @@ pub unsafe fn access(&self) -> &T { /// /// Callers must ensure that there are no more concurrent users of the= revocable object. unsafe fn revoke_internal(&self) -> bool { - let revoke =3D self.is_available.swap(false, Ordering::Relaxed); + let revoke =3D self.is_available.xchg(false, Relaxed); =20 if revoke { if SYNC { --=20 2.50.1 (Apple Git-155)