From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 90B4A3859CB for ; Fri, 7 Aug 2026 07:02:28 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086149; cv=none; b=BK0YNm36uDbxTWPR2vDkQ0yC3b00JMw6VfzhIaPZFJbpsdgjO7Kq9kmkV5UvhbThzV1SSK0tYDJWnlAgU6iflJFiu1b+5hZCNdmaQN6iJqfNW8+Weosh9c6JwI4ZySwTr2x8pCVoOzmErfZalr3Beo4Dw08njeQgvlCcN6T1lxg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086149; c=relaxed/simple; bh=6ESVuJjZ9ui3/4DwNjKV/PraMIkzSH0Vd92IDm2WfD8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=EOvb2YXjvv2ti6hbTMH3MduH6HZCXkbTun74DzrP0fLmz5Mc6lAvDS0r3CgwrURK3oNU7bpik38Ze6EoXwl6ND+5+mBWbo8uxHvNAtydptYtMNlF1ViYJiJ9FlW9Hbe6nbPOFr84xPdeAXbMhYDsHOfpS+X0QrK33FRioKd2PEA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bS0DasBk; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bS0DasBk" Received: by smtp.kernel.org (Postfix) with ESMTPSA id AA8531F00A3D; Fri, 7 Aug 2026 07:02:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086148; bh=3oQWJsZRvEwPxCRQ6qISEKyc40r/iISGRFtA3wGNP7M=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=bS0DasBk0ktCm6tC1qyv12CkzWVF+LsAx+u0C0vrigp7NuWL9Kl1dmT06geswfrc5 JUX8KdGtIxADu/fHOpvp9Yd+kEa90/iwXIFuOrrotN01VrYnvpToyabOqwfpV4iZSU vdxx4t7WI0BjIXGkoRv7RTCmw5VAQnOmJxdr8nUe/M4Nnk5cO6cIC1tb03liYb1Jol S/DwsauEejATU5Bd/z6832I5NtHAqiRUhYDF9S6IK6LdgMmhBkHUpCQSueGtQOsQBs J4CDrbsl2Pz0YdODLsjMk2xAuNx0h4PtqyubwVxauedL9lx1F6+kwrtQ1vygqFe2+U PD7px93/fqx4w== Received: from phl-compute-03.internal (phl-compute-03.internal [10.202.2.43]) by mailfauth.phl.internal (Postfix) with ESMTP id C3C6DF40068; Fri, 7 Aug 2026 03:02:26 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-03.internal (MEProxy); Fri, 07 Aug 2026 03:02:26 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGymVtxh0AmopRY+TfTfXYPCmloL5eVBz8w4y+3KFrHM+Bwv6o50auh1636bSBYkE H90J+8ZvDwGP+GEnsL03hmghhkCvgcLvZRJlD0Z0K6idzeBe96fVCfVF90zAumgR10DH6A dzrkds6t6yssM5k8WnRKyJY7PoN9yPTJEd8yclKHhKcjgA77IEZ54Ob7IjTR0RbJw1Jd/H MAKXnEz1+HSwhiWHnu3/dGxtHg6i6x+jlrZPffCsJNEQERYySXee/h/1gQg2z905mRcC4Y mB/oQJoiVpI6TFSRaftlL1E+wAZyQH5fXLMWv+6L12SLyEAsyE0kkUaQtsxqspvPUOq9AA jTmfS5/QKolSD4R7HDa4dhZQPCPAQYRI1iO+O7rxtEWugxuOlzZllTY7KeKdL0DiSaLNZF eSGMRKMDR1xjY6X4KybfHsyb4fEYt/A25awL5+AfR+6blU5KwI+gwGYCsvfirAgfg9uJmu U9SNSfDoBdVOIztvn7gRHPjFYb88EDs+koyXI7iFHUSgjNg4R9XeK0W+X3LGWhJ2zTYqU1 c9tCwDptqYBijtzsT+E3SNviqTdFE2KA8m1eMp/pumKG+QlCp8UiP6Z4dwy8cLP957BsYq EiMPGVrpuUfZ+Q2Jf5UB9r/qSlm+dBuKFyXGd07YyY59dUXQ+lbeCCVNhm8g X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:26 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" , Joel Fernandes Subject: [PATCH v5 01/18] preempt: Track NMI nesting to separate per-CPU counter Date: Fri, 7 Aug 2026 00:01:58 -0700 Message-ID: <20260807070218.27144-2-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Joel Fernandes Move NMI nesting tracking from the preempt_count bits to a separate per-CPU counter (nmi_nesting). This is to free up the NMI bits in the preempt_count, allowing those bits to be repurposed for other uses. Reduce NMI_BITS from 4 to 1, using it only to detect if we're in an NMI. The per-CPU counter currently caps nesting at 15. [boqun: Address Steven Rostedt's comment on the BUG_ON() condition] [boqun: Use preempt_count_set() in __nmi_exit() to avoid underflow] Suggested-by: Boqun Feng Signed-off-by: Joel Fernandes Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng --- include/linux/hardirq.h | 17 +++++++++++++---- include/linux/preempt.h | 9 +++++++-- kernel/softirq.c | 2 ++ tools/testing/selftests/bpf/bpf_experimental.h | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index d57cab4d4c06..8d4895531a45 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -10,6 +10,8 @@ #include #include =20 +DECLARE_PER_CPU(unsigned int, nmi_nesting); + extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); =20 @@ -102,14 +104,17 @@ void irq_exit_rcu(void); */ =20 /* - * nmi_enter() can nest up to 15 times; see NMI_BITS. + * nmi_enter() can nest - nesting is tracked in a per-CPU counter. */ #define __nmi_enter() \ do { \ lockdep_off(); \ arch_nmi_enter(); \ - BUG_ON(in_nmi() =3D=3D NMI_MASK); \ - __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ + __this_cpu_inc(nmi_nesting); \ + __preempt_count_add(HARDIRQ_OFFSET); \ + preempt_count_set(preempt_count() | NMI_MASK); \ } while (0) =20 #define nmi_enter() \ @@ -124,8 +129,12 @@ void irq_exit_rcu(void); =20 #define __nmi_exit() \ do { \ + unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + nesting =3D __this_cpu_dec_return(nmi_nesting); \ + if (!nesting) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index d964f965c8ff..586f96688325 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,6 +17,8 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) + * - bits 16-19 are the hardirq count (max # of hardirqs: 16) + * - bit 20 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -24,16 +26,19 @@ * there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * + * NMI nesting depth is tracked in a separate per-CPU variable + * (nmi_nesting) to save bits in preempt_count. + * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00f00000 + * NMI_MASK: 0x00100000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) diff --git a/kernel/softirq.c b/kernel/softirq.c index 4425d8dce44b..10af5ed859e7 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -88,6 +88,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif =20 +DEFINE_PER_CPU(unsigned int, nmi_nesting); + /* * SOFTIRQ_OFFSET usage: * diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing= /selftests/bpf/bpf_experimental.h index 67ff7882299e..e4e12001fce9 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -367,7 +367,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup,= const char *name__str, #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 00FCC390CBE; Fri, 7 Aug 2026 07:02:29 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086151; cv=none; b=PPPvPBMQPhsh1mFpW73RsAG6VGgSQQeEOQHREjEX9ynJ/5NGR4t2aYQj3pZT1cvO2QLBANQCdRZGxgyaWePxX3DRELjrmvjqDkxiB3fxR+OU2WV1awpGQ4Jn7mhNqvfeRdy6ddHsBgyF4kFc+aJXMlv2D0Xe/MUpoGxLv6/Kc3E= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086151; c=relaxed/simple; bh=CEFovGyeXMpftokmGqMA9XIJ9FL21tUPcuXNNde++wI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=sR3HTWv4/HtmbcDGDwYikFIss/x5DJpm0o1ERiFr6rarvZrP/YRbXJf3HzDIR00RXeDCZIUjgnzVRxn78HzdwjvTNB1SaPqoTFslu0JoDM5KkqPRAPPXYrMqbhpxwcrEDJDKFZFMfL9iVJ0wNxk3dVwJ/kGXtecpWA1I1Saq1dA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=c13Wuymp; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="c13Wuymp" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1A0351F00A3F; Fri, 7 Aug 2026 07:02:29 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086149; bh=VBb3FbGLQGm67Ub08TQMzCeI8jjhoUtbqsCyiVOt/HM=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=c13WuympHNd2JKDLxnNQjqqe2JW+0ut5J7pjd3jjRqaICvEq94dijK1d7x91YKEKQ vlOu0k4JmHHTGCwscyntwsRexGDKGDfSh5kJITJI1UA9GzWjjY6sM71TLA2fg19R4u I9NLZfbBz4sP3KuEkEgVPf/AoIi6B4wDL9/LeD7LaFb2w18jVVmb70/qi6+qu2HTH0 XXuxOt37mPSvloTWfEuwbrmUrcMjLe/3L3fowatVqCv1rDwHv4XvtJGSjYkJCg3v8x L7MnjYDR2/MqTzAjc+gg1HVyA+jxfdV2vnURgh0IJQRRyIKkIABAMhddjucquMvnaE co/9Uq5X7rRYw== Received: from phl-compute-08.internal (phl-compute-08.internal [10.202.2.48]) by mailfauth.phl.internal (Postfix) with ESMTP id 31C7AF40066; Fri, 7 Aug 2026 03:02:28 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-08.internal (MEProxy); Fri, 07 Aug 2026 03:02:28 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTELT6mPBN+tf+gYDjbmfxF2aJuNia3fHRbo+cksodwgHpv175RY/La8br7LVuLKD+ xpTrulxkYenRUqjuFjoYpWp5kB81fkgSYs4cUOn9Aimg2+sWPDBZPSJD0AkLNVKo9ymnEn XwkNJDHucJ7KY3ALZ5VcWOWe5XHuNtoJV5e7Q54FFSKl7KOd+nrEm1L6dR4Jdu+LjwwV/e gMJ05NqNZsVzzu9M3eDFzAUAJPeoh4s2jJHbhIzGNi7DAZxkNRx1H6hWOvNKjwmDuMbW1g 3/Yvg82sjVAOqgIP9o+xyvh3ddqbO1HSSPQ13n3n5YTt3W/2pxiARls47VLAs4X6PdSfRw 0i4flKekFgwscxxBdQcR10/AiGpVK7ZvcpqdyKp8fFzbNPXbAWO/O6dkRe9wI3DbEwamyj 1ljajEO1BLGPRaKXrzCC3Il85ddUqjAOJ7MY+hIJzUIaL33rHMw7RI5OSRSrGmdLV+4uX6 omIrstDim84VQSnHYjMwxAjGLppu8ObQ+u9VgF1Olbw6+ZaEQtvdrtmruNjt4hfKaSNddW hCJIvye6IAIJpQ5oRMt9vPKlzLcBVqHIQL1wm4m+CCuchOgYfinLzD1CQdWBqX7XGqxw6j g/jEoByw6E4l4r3w+sdVdTxBJDgO+uMTia6eko+FUwMEsEURRplm/oC/xL0A X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:27 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 02/18] preempt: Introduce HARDIRQ_DISABLE_BITS Date: Fri, 7 Aug 2026 00:01:59 -0700 Message-ID: <20260807070218.27144-3-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In order to support preempt_disable()-like interrupt disabling, that is, using part of preempt_count() to track interrupt disabling nesting level, change the preempt_count() layout to contain 8-bit HARDIRQ_DISABLE count. Co-developed-by: Lyude Paul Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng --- include/linux/preempt.h | 16 +++++++++++----- tools/testing/selftests/bpf/bpf_experimental.h | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 586f96688325..e2d3079d3f5f 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,8 +17,9 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) - * - bits 16-19 are the hardirq count (max # of hardirqs: 16) - * - bit 20 is the NMI flag (no nesting count, tracked separately) + * - bits 16-23 are the hardirq disable count (max # of hardirq disable: 2= 56) + * - bits 24-27 are the hardirq count (max # of hardirqs: 16) + * - bit 28 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -31,29 +32,34 @@ * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 - * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00100000 + * HARDIRQ_DISABLE_MASK: 0x00ff0000 + * HARDIRQ_MASK: 0x0f000000 + * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 #define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) =20 #define __IRQ_MASK(x) ((1UL << (x))-1) =20 #define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_= DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) =20 #define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT) #define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) #define NMI_OFFSET (1UL << NMI_SHIFT) =20 diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing= /selftests/bpf/bpf_experimental.h index e4e12001fce9..0159a3d365c8 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -366,17 +366,20 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgrou= p, const char *name__str, =20 #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 #define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) =20 #define __IRQ_MASK(x) ((1UL << (x))-1) =20 #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_= DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) =20 --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7439F390987 for ; Fri, 7 Aug 2026 07:02:31 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086152; cv=none; b=bIWa40+1bVK2a5F2fKJTbrQoAp3gk12JQGdnQmv60swDuoEwAxddnOaZbSoq4Amw7WnVrayAFwchkmc1f9bcUJEs2FmpX8GXsnqHEzLz/CI9uNs/9PCzPub6Wtw4c3QdVv7KMPIk29fOnFURvodJlRrQiV52p/aWcHX3zTVGq28= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086152; c=relaxed/simple; bh=ZdmP9HCAd2oDU8aGybcrgEt64oPFu3q8ZuAOXbDuT/Y=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=hQ+/HF1bb11RGfvDDLqPSMRDwT0vA4LRmD3v+rOqKsrcsqzeA8A03Rrwrbt3v8fwV/lZq/xHDh+j5805LCKYamseraapo27YU+2G3IgVDRQcRdXtWTUHAFyr30FSqtywf24DEmr0NLXm8M2eklcH5/Y+/XoOIlWSQa3wJvaVYw4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Np8sEcY4; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Np8sEcY4" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 83CD01F00A3A; Fri, 7 Aug 2026 07:02:30 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086151; bh=4/3Q4RbdeBoEcapTLQLRtJHDgKIOcLItV3asUF3+BOc=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Np8sEcY4ZDzBl03bhMcGs0D4dixG36SCyzIvLvudXi97D37PI+mv5cdBIhc72fIb2 /SGJpsTolQW4vjYMp/bB4pxXdpvAG1kuBG8IpP7UqfP56fpl1SxBapADRkaoCJdzb6 eySO2J+FZ9jLBuW+CNnaCCL+TBm9QpVpBEWxYOZOvgW9D5XT8qpnuvxxvuiDhW3v3Z AALULd8L1ToF7dCEGnX8jCJVa868QhIvAqr1za9Em5UsxNp0aLv/ypJT3hpCb3RogO QAZTGAf1iyGI0ZsX1S2PfdCa34tVM2ZWru3wFA1Al0wLQlLuRUtSy6N3AYd5/h2lOp F6D3bCr0G64TQ== Received: from phl-compute-08.internal (phl-compute-08.internal [10.202.2.48]) by mailfauth.phl.internal (Postfix) with ESMTP id 9B536F40067; Fri, 7 Aug 2026 03:02:29 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-08.internal (MEProxy); Fri, 07 Aug 2026 03:02:29 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGymVtxh0AmopRY+TfTfXYPCmloL5eVBz8w4y+3KFrHM+Bwv6o50auh1636bSBYkE H90J+8ZvDwGP+GEnsL03hmghhkCvgcLvZRJlD0Z0K6idzeBe96fVCfVF90zAumgR10DH6A dzrkds6t6yssM5k8WnRKyJY7PoN9yPTJEd8yclKHhKcjgA77IEZ54Ob7IjTR0RbJw1Jd/H MAKXnEz1+HSwhiWHnu3/dGxtHg6i6x+jlrZPffCsJNEQERYySXee/h/1gQg2z905mRcC4Y mB/oQJoiVpI6TFSRaftlL1E+wAZyQH5fXLMWv+6L12SLyEAsyE0kkUaQtsxqspvPUOq9Dg i49SfvrWIX/QS8rpDl7olpqYaPYYioefeVoSw6nr0ibiHzViZMB6HA0ao5Z2T9HFRzl3Fk rylAAaFQ+8yMbU+thtcnsWamiAlUp7atxU73NAtXVpXfW7hIagKa7jPxqc6ILyElp9CLkM 4wK1WGoAHeLiOwiif2ii82XWuI8Qyjcx603f6Gi1C18cmlxIIgpRVLQmYU9rTGDxjd1ZyW 8hF4NjlwKZ2MRjm9RMdM5aowqQ3stc5AuybFvyvEde9KblUG5xQ4DW79W3LTICfkDhWhUs Ft5r2DdAJIr3FlF9ZmHw7ELzzyhd8qcnvWdXSqhtn97WstkMcuC3u+pcnrjg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:29 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" , Heiko Carstens Subject: [PATCH v5 03/18] preempt: Introduce __preempt_count_{sub,add}_return() Date: Fri, 7 Aug 2026 00:02:00 -0700 Message-ID: <20260807070218.27144-4-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In order to use preempt_count() to track the interrupt disable nesting level, __preempt_count_{add,sub}_return() are introduced, as their names suggest, these primitives return the new value of the preempt_count() after changing it. The following example shows the usage of it in local_interrupt_disable(): // increase the HARDIRQ_DISABLE bit new_count =3D __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET); // if it's the first-time increment, then disable the interrupt // at hardware level. if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D HARDIRQ_DISABLE_OFFSET) { local_irq_save(flags); raw_cpu_write(local_interrupt_disable_state, flags); } Having these primitives will avoid a read of preempt_count() after changing preempt_count() on certain architectures. Acked-by: Heiko Carstens # s390 Signed-off-by: Boqun Feng --- arch/arm64/include/asm/preempt.h | 20 ++++++++++++++++++++ arch/s390/include/asm/preempt.h | 10 ++++++++++ arch/x86/include/asm/preempt.h | 10 ++++++++++ include/asm-generic/preempt.h | 14 ++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/pree= mpt.h index 932ea4b62042..9ecc2766a9f2 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -55,6 +55,26 @@ static inline void __preempt_count_sub(int val) WRITE_ONCE(current_thread_info()->preempt.count, pc); } =20 +static inline int __preempt_count_add_return(int val) +{ + u32 pc =3D READ_ONCE(current_thread_info()->preempt.count); + + pc +=3D val; + WRITE_ONCE(current_thread_info()->preempt.count, pc); + + return pc; +} + +static inline int __preempt_count_sub_return(int val) +{ + u32 pc =3D READ_ONCE(current_thread_info()->preempt.count); + + pc -=3D val; + WRITE_ONCE(current_thread_info()->preempt.count, pc); + + return pc; +} + static inline bool __preempt_count_dec_and_test(void) { struct thread_info *ti =3D current_thread_info(); diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preemp= t.h index 6e5821bb047e..0a25d4648b4c 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -139,6 +139,16 @@ static __always_inline bool should_resched(int preempt= _offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) =3D=3D preempt_of= fset); } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + return val + __atomic_add(val, &get_lowcore()->preempt_count); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return __preempt_count_add_return(-val); +} + #define init_task_preempt_count(p) do { } while (0) /* Deferred to CPU bringup time */ #define init_idle_preempt_count(p, cpu) do { } while (0) diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 578441db09f0..1220656f3370 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -85,6 +85,16 @@ static __always_inline void __preempt_count_sub(int val) raw_cpu_add_4(__preempt_count, -val); } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + return raw_cpu_add_return_4(__preempt_count, val); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return raw_cpu_add_return_4(__preempt_count, -val); +} + /* * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to resch= edule * a decrement which hits zero means we have no preempt_count and should diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index 51f8f3881523..c8683c046615 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -59,6 +59,20 @@ static __always_inline void __preempt_count_sub(int val) *preempt_count_ptr() -=3D val; } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + *preempt_count_ptr() +=3D val; + + return *preempt_count_ptr(); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + *preempt_count_ptr() -=3D val; + + return *preempt_count_ptr(); +} + static __always_inline bool __preempt_count_dec_and_test(void) { /* --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A5882388885; Fri, 7 Aug 2026 07:02:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086153; cv=none; b=EFig9rSIRpq3AtJTWJDUAVGpblCPF9p43md74cZU3CecXQUN9iEKJurwOIL7wTe08gYBE6zn/pwDhuCLWOHeXWX6D8mf7+f2IItzg5QqqTPgSIMcQfBua1M2v97Qaa8y/uSjq8FagZeGW7V00ABH9bLyDJ2WkgoSuJvjqMbBFiA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086153; c=relaxed/simple; bh=X8RZfaXpnjo3eZ8Yt9kgyqp+ytrFOxLZYmWiV2ArpVs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=WxVWtygYckLshvZFtCtM1QTC71RQtXdvmMvHmhLHK0w5zhtiDoHBtTptZf7tgGn2epA/qHhsxcYMByxxlDW6CL1WQv6p/q62VhtxNFA9FhKEuxXUNI9bQcDLHgKHvyeGVPD7P06Few//7d5as2TcIcpVXgWBipM346nIbQSeTYE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=J97iG+HV; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="J97iG+HV" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 035151F000E9; Fri, 7 Aug 2026 07:02:31 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086152; bh=Q/FRi4i2Q2qXDY7wTkxJ2+HYMjzwJRaRncVAKmZRve4=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=J97iG+HVH08bA6/oUKqy2GyoquKMHzCZdai6FnIk6qVhV8aTPOnXmGrjGu0eqte7S m3Z++xHLFcFMxEgyXe4Pzvf2g/4vnU5dG7N8CWLV5VUaWvlemYndVyrEE9cvTEIeEB Ih7jENZ9PrGLucc5XJXHboMs2iZiP38X3vBQeQ7AhUjFVRU6LIAgXGvGLkrFMN9dmt kZ6b1beCGAkWaOPw2a4ElVhxmra+D15nOquhhnkM84N4mPVBWEOgBn5WB1cPBooTSM 37Tp+seWRylc3JEU855CAzDXcbkSC9LCi3T76hS0Vy6ELNF5tjgyAt+akmjRcXmR0K ge7kbQN2VrS4Q== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 1CD1DF40066; Fri, 7 Aug 2026 03:02:31 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Fri, 07 Aug 2026 03:02:31 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGymVtxh0AmopRY+TfTfXYPCmloL5eVBz8w4y+3KFrHM+Bwv6o50auh1636bSBYkE H90J+8ZvDwGP+GEnsL03hmghhkCvgcLvZRJlD0Z0K6idzeBe96fVCfVF90zAumgR10DH6A dzrkds6t6yssM5k8WnRKyJY7PoN9yPTJEd8yclKHhKcjgA77IEZ54Ob7IjTR0RbJw1Jd/H MAKXnEz1+HSwhiWHnu3/dGxtHg6i6x+jlrZPffCsJNEQERYySXee/h/1gQg2z905mRcC4Y mB/oQJoiVpI6TFSRaftlL1E+wAZyQH5fXLMWv+6L12SLyEAsyE0kkUaQtsxqspvPUOq9Vn O2zmne8NrlnW/YvyJ9H9pWzAjjivyTHsSfPmcutkdMCiQefr+zO7f0iq/BFYmzpmkrIrqB xw9Mq2yh5Rcs8TJcoUfIWlqMr1wlPIch825HR6eKYR3vOyXiQ3/vF0rll2PccyV/TnaRF9 3XPyxm3EkRwJGCqUO3HVRvx+m9a1RKZLhjPU2NOXjrgvevzkpZpFhO38BU8CvkUlHy298y SlP8tS5cbB2BO7bMEbctUoP33YiUd8UwmfKd+mbgsRH2FTaPQ7kByEmFZgR3Ayz2dQugFZ Z/yPJpz+PoFenJXwYoPbO1xJKNV/re3T0bPsiI8c9zAxlTp6y1oNOEnq+ubg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:30 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" , Stafford Horne Subject: [PATCH v5 04/18] openrisc: Include in smp.h Date: Fri, 7 Aug 2026 00:02:01 -0700 Message-ID: <20260807070218.27144-5-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul While OpenRISC currently doesn't fail to build upstream, it appears that including in the right headers is enough to break that - primarily because OpenRISC's asm/smp.h header doesn't actually provide any definition for struct cpumask. Which means the only reason we aren't failing to build the kernel is because we've been lucky enough that every spot including asm/smp.h already has definitions for struct cpumask pulled in. This became evident when trying to work on a patch series for adding ref-counted interrupt enable/disable to the kernel, where introducing a new interrupt_rc.h header suddenly introduced a build error on OpenRISC: In file included from include/linux/interrupt_rc.h:17, from include/linux/spinlock.h:60, from include/linux/mmzone.h:8, from include/linux/gfp.h:7, from include/linux/mm.h:7, from arch/openrisc/include/asm/pgalloc.h:20, from arch/openrisc/include/asm/io.h:18, from include/linux/io.h:12, from drivers/irqchip/irq-ompic.c:61: arch/openrisc/include/asm/smp.h:21:59: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration 21 | extern void arch_send_call_function_ipi_mask(const struct cpum= ask *mask); | ^~~~= ~~~ arch/openrisc/include/asm/smp.h:23:54: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration 23 | extern void set_smp_cross_call(void (*)(const struct cpumask *= , unsigned int)); | ^~~~~~~ drivers/irqchip/irq-ompic.c: In function 'ompic_of_init': >> drivers/irqchip/irq-ompic.c:191:28: error: passing argument 1 of 'set_smp_cross_call' from incompatible pointer type [-Werror=3Dincompatible-pointer-types] 191 | set_smp_cross_call(ompic_raise_softirq); | ^~~~~~~~~~~~~~~~~~~ | | | void (*)(const struct cpumask *, un= signed int) arch/openrisc/include/asm/smp.h:23:32: note: expected 'void (*)(const struct cpumask *, unsigned int)' but argument is of type 'void (*)(const struct cpumask *, unsigned int)' 23 | extern void set_smp_cross_call(void (*)(const struct cpumask *= , unsigned int)); To fix this, let's take an example from the smp.h headers of other architectures (x86, hexagon, arm64, probably more): just include linux/cpumask.h at the top. Signed-off-by: Lyude Paul Acked-by: Stafford Horne Signed-off-by: Boqun Feng --- arch/openrisc/include/asm/smp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/openrisc/include/asm/smp.h b/arch/openrisc/include/asm/sm= p.h index 007296f160ef..84653aaffa96 100644 --- a/arch/openrisc/include/asm/smp.h +++ b/arch/openrisc/include/asm/smp.h @@ -9,6 +9,8 @@ #ifndef __ASM_OPENRISC_SMP_H #define __ASM_OPENRISC_SMP_H =20 +#include + #include #include =20 --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BBDDC3A6EE9; Fri, 7 Aug 2026 07:02:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086156; cv=none; b=CMkAYn2tfnPMqnkQY9MKZMcGKKo8mrGazHbpUJUWK9tjxB9Z2c9VVou6MDEBi0ClpSD7kRF+siYseYptfiYLohvPu9NVcpAhPAxxDtrUbonizTMWAsKd3KIWIXqE34VqgSltrpqevS//sbMujgVa/jo5eQz+SUtfH7T0mj5JsSM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086156; c=relaxed/simple; bh=PVXyqxhseUmydi1eisCdOsaQNFzvMbw33zCCwC1Pfc4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tUC/GBUy7G9ujy4lj4xrOIqc9LdZjSKZZ5n5TaNi/nY2lQ5A2E5PiZ+eZdBKy4m3wFU1DRL3+gv58rb1hEo3sqAFVHI3e1R9Jl1wEXDaLNrIuTA79fkDUfV6o0D8Fi59lL6NVY8L2zwmruRq/X06VovF72j2Kh0nTyWs3RAtwoc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bzjkLweF; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bzjkLweF" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 8EA451F00A3D; Fri, 7 Aug 2026 07:02:33 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086154; bh=T09HNyKh2goYoTDqpZTcXQ875bAGppLWcTT0g6z87kI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=bzjkLweFK2dTq+a0MxixQQwX31GmbaJ8QTNpnHS0EMUiABZ05JnKbzq4p7hWuhwk5 Y7eIvOW+D+DHBKkk+N9J5b6sFcDccefuzhqwDRu+4Hz80sG0ufmfyFryzzIe3tkd0j 7lcZwG/M3tnQgz0h8gxpEhEemc2y3XWOBfKOs624ANZkN8/6nbf2yX+FUp16DOyRc0 QA+YA4Tz6Tt7OPwKWkGYLg7AzOg2t0uflFXBqwaqNG07Rl06W41zzdZiRRSPUjxF7I 2iRkD5EnBMCEHI2fMe4hUfGob7WQRlo+eccW99JGIp8ssxLU1QW+5CgVtQqdXCfwed pUUStr4HLy13Q== Received: from phl-compute-01.internal (phl-compute-01.internal [10.202.2.41]) by mailfauth.phl.internal (Postfix) with ESMTP id A65BDF40067; Fri, 7 Aug 2026 03:02:32 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-01.internal (MEProxy); Fri, 07 Aug 2026 03:02:32 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTELT6mPBN+tf+gYDjbmfxF2aJuNia3fHRbo+cksodwgHpv175RY/La8br7LVuLKD+ xpTrulxkYenRUqjuFjoYpWp5kB81fkgSYs4cUOn9Aimg2+sWPDBZPSJD0AkLNVKo9ymnEn XwkNJDHucJ7KY3ALZ5VcWOWe5XHuNtoJV5e7Q54FFSKl7KOd+nrEm1L6dR4Jdu+LjwwV/e gMJ05NqNZsVzzu9M3eDFzAUAJPeoh4s2jJHbhIzGNi7DAZxkNRx1H6hWOvNKjwmDuMbW1g 3/Yvg82sjVAOqgIP9o+xyvh3ddqbO1HSSPQ13n3n5YTt3W/2pxiARls47VLAs4X6PdSfpg gq+3lRQ3VEvCnMZEGEL7pINdUzVaq3Z8m6ruK1F/giVFo7EOoWyD6lx/OtSvrfkle/0D8E 2WlEe0tVfxXNtr+BHYU7ZOIYRLfnNYaj4JkK5cisBSF0OTRHwvT+mg4as/H0WdkwJjT3HB MokMcGZTPQpnxNVZlullvNKRdpP39ZCTmsSNu2D2qp/i6II4VxdVkhDepDQWqLqHQ1pJlj ZhPIowBEKwxB7l4e4fOCJvw6Yralr66+9WPYkYQW3mTUxDDyTA/1AdKeLnd8NkH8ESKbTH bKtasAN59FABDYCnSZueq+RpnZlvUM3EtizOVlTIv58bj//sHkjR8vkCaaGw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:32 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 05/18] irq & spin_lock: Add counted interrupt disabling/enabling Date: Fri, 7 Aug 2026 00:02:02 -0700 Message-ID: <20260807070218.27144-6-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently the nested interrupt disabling and enabling is represented by _irqsave() and _irqrestore() APIs, which are relatively unsafe, for example: spin_lock_irqsave(l1, flag1); spin_lock_irqsave(l2, flag2); spin_unlock_irqrestore(l1, flags1); // accesses to interrupt-disable protected data will cause races This is even easier to trigger with guard facilities: unsigned long flag2; scoped_guard(spin_lock_irqsave, l1) { spin_lock_irqsave(l2, flag2); } // l2 locked but interrupts are enabled. spin_unlock_irqrestore(l2, flag2); (Hand-to-hand locking critical sections are not uncommon for a fine-grained lock design) And because of this unsafety, Rust cannot easily wrap the interrupt-disabling locks in a safe API, which complicates the design. To resolve this, introduce a new set of interrupt disabling APIs: * local_interrupt_disable(); * local_interrupt_enable(); They work like local_irq_save() and local_irq_restore() except that 1) the outermost local_interrupt_disable() call saves the interrupt state into a per-CPU variable, so that the outermost local_interrupt_enable() can restore the state, and 2) a per-CPU counter is added to record the nest level of these calls, so that interrupts are not accidentally enabled inside the outermost critical section. Also add the corresponding spin_lock primitives: spin_lock_irq_disable() and spin_unlock_irq_enable(), as a result, code as follows: spin_lock_irq_disable(l1); spin_lock_irq_disable(l2); spin_unlock_irq_enable(l1); // Interrupts are still disabled. spin_unlock_irq_enable(l2); doesn't have the issue that interrupts are accidentally enabled. This also makes the wrapper of interrupt-disabling locks on Rust easier to design. Co-developed-by: Lyude Paul Signed-off-by: Lyude Paul [boqun: Apply Peter's feedback and fix spell errors reported by Ingo] [boqun: Address the duplicate spin_acquire() spotted by sashiko] Signed-off-by: Boqun Feng --- include/linux/interrupt_rc.h | 95 ++++++++++++++++++++++++++++++++ include/linux/preempt.h | 4 ++ include/linux/spinlock.h | 23 ++++++++ include/linux/spinlock_api_smp.h | 41 ++++++++++++++ include/linux/spinlock_api_up.h | 15 +++++ include/linux/spinlock_rt.h | 18 ++++++ kernel/locking/spinlock.c | 31 +++++++++++ kernel/softirq.c | 28 +++++++++- 8 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 include/linux/interrupt_rc.h diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h new file mode 100644 index 000000000000..bb62979270a6 --- /dev/null +++ b/include/linux/interrupt_rc.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_INTERRUPT_RC_H +#define __LINUX_INTERRUPT_RC_H + +/* + * include/linux/interrupt_rc.h - refcounted local processor interrupt + * management. + * + * Since the implementation of this API currently depends on + * local_irq_save()/local_irq_restore(), we split this into its own header= to + * make it easier to include without hitting circular header dependencies. + */ + +#include +#include +#include +#include +#include + +#ifndef MODULE +/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(= ). */ +DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); + +static __always_inline void __local_interrupt_disable(void) +{ + unsigned long flags; + + local_irq_save(flags); + raw_cpu_write(local_interrupt_disable_state, flags); +} + +static __always_inline void __local_interrupt_enable(void) +{ + unsigned long flags =3D raw_cpu_read(local_interrupt_disable_state); + + local_irq_restore(flags); +} + +#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE +static __always_inline void _local_interrupt_disable(void) +{ + __local_interrupt_disable(); +} + +static __always_inline void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +#else +extern void _local_interrupt_disable(void); +extern void _local_interrupt_enable(void); +#endif + +#else /* !MODULE */ +extern void _local_interrupt_disable(void); +extern void _local_interrupt_enable(void); +#endif /* !MODULE */ + +static inline void local_interrupt_disable(void) +{ + int new_count; + + WARN_ON_ONCE(in_nmi()); + + new_count =3D hardirq_disable_enter(); + + /* Is hardirq disable count overflow soon? */ + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) && + DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) + + (10 << HARDIRQ_DISABLE_SHIFT) >=3D + HARDIRQ_DISABLE_MASK)) + return; + + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ + + if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D HARDIRQ_DISABLE_OFFSET) + _local_interrupt_disable(); +} + +static inline void local_interrupt_enable(void) +{ + int new_count; + + /* Unpaired local_interrupt_enable()? Warn and abort. */ + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) && + DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) =3D=3D 0= )) + return; + + new_count =3D hardirq_disable_exit(); + + if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D 0) + _local_interrupt_enable(); +} + +#endif /* !__LINUX_INTERRUPT_RC_H */ diff --git a/include/linux/preempt.h b/include/linux/preempt.h index e2d3079d3f5f..33fc4c814a9f 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context= _level(void) #define in_softirq() (softirq_count()) #define in_interrupt() (irq_count()) =20 +#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) = >> HARDIRQ_DISABLE_SHIFT) +#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE= _OFFSET) +#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_= OFFSET) + /* * The preempt_count offset after preempt_disable(); */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 241277cd34cf..3d405cc4c121 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -57,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *= lock) __releases(lock) #endif =20 #define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock) #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock) #define raw_spin_unlock(lock) _raw_spin_unlock(lock) #define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock) +#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock) =20 #define raw_spin_unlock_irqrestore(lock, flags) \ do { \ @@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *l= ock) __releases(lock) =20 #define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lo= ck, &(flags)) =20 +#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(l= ock) + #ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ #include @@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *= lock) raw_spin_lock_irq(&lock->rlock); } =20 +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + raw_spin_lock_irq_disable(&lock->rlock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ raw_spin_lock_irqsave(spinlock_check(lock), flags); \ @@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t= *lock) raw_spin_unlock_irq(&lock->rlock); } =20 +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) __no_context_analysis +{ + raw_spin_unlock_irq_enable(&lock->rlock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsig= ned long flags) __releases(lock) __no_context_analysis { @@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spin= lock_t *lock, unsigned lon } #define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(fl= ags)) =20 +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) __no_context_analysis +{ + return raw_spin_trylock_irq_disable(&lock->rlock); +} + /** * spin_is_locked() - Check whether a spinlock is locked. * @lock: Pointer to the spinlock. diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_= smp.h index bda5e7a390cd..90909d933dab 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct loc= kdep_map *map) void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock); void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) __acquires(lock); +void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock); =20 unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) __acquires(lock); @@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock)= __cond_acquires(true, void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock= ); +void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releas= es(lock); void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) __releases(lock); @@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsign= ed long flags) #define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock) #endif =20 +/* Use the same config as spin_lock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ +#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE #define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock) #endif @@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsign= ed long flags) #define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock) #endif =20 +/* Use the same config as spin_unlock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ +#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(loc= k) +#endif + #ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE #define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrest= ore(lock, flags) #endif @@ -105,6 +118,16 @@ static __always_inline bool _raw_spin_trylock_irq(raw_= spinlock_t *lock) return false; } =20 +static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *= lock) + __cond_acquires(true, lock) +{ + local_interrupt_disable(); + if (_raw_spin_trylock(lock)) + return true; + local_interrupt_enable(); + return false; +} + static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock= , unsigned long *flags) __cond_acquires(true, lock) { @@ -143,6 +166,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t = *lock) LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); } =20 +static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + local_interrupt_disable(); + preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); + LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); +} + static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock) __no_context_analysis { @@ -188,6 +220,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_= t *lock) preempt_enable(); } =20 +static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock) + __releases(lock) +{ + spin_release(&lock->dep_map, _RET_IP_); + do_raw_spin_unlock(lock); + local_interrupt_enable(); + preempt_enable(); +} + static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock) { diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_u= p.h index a9d5c7c66e03..d03d3065ee04 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -42,6 +42,9 @@ #define __LOCK_IRQSAVE(lock, flags, ...) \ do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0) =20 +#define __LOCK_IRQ_DISABLE(lock, ...) \ + do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0) + #define ___UNLOCK_(lock) \ do { __release(lock); (void)(lock); } while (0) =20 @@ -61,6 +64,9 @@ #define __UNLOCK_IRQRESTORE(lock, flags, ...) \ do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0) =20 +#define __UNLOCK_IRQ_ENABLE(lock, ...) \ + do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0) + #define _raw_spin_lock(lock) __LOCK(lock) #define _raw_spin_lock_nested(lock, subclass) __LOCK(lock) #define _raw_read_lock(lock) __LOCK(lock, shared) @@ -70,6 +76,7 @@ #define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared) #define _raw_write_lock_bh(lock) __LOCK_BH(lock) #define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock) +#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock) #define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared) #define _raw_write_lock_irq(lock) __LOCK_IRQ(lock) #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) @@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_sp= inlock_t *lock) return 1; } =20 +static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *l= ock) + __cond_acquires(true, lock) +{ + __LOCK_IRQ_DISABLE(lock); + return 1; +} + static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock,= unsigned long *flags) __cond_acquires(true, lock) { @@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(r= wlock_t *lock, unsigned l #define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock) #define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared) #define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock) +#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock) #define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared) #define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock) #define _raw_spin_unlock_irqrestore(lock, flags) \ diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 373618a4243c..560d06384e0c 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lo= ck) rt_spin_lock(lock); } =20 +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) +{ + rt_spin_lock(lock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ @@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t= *lock) rt_spin_unlock(lock); } =20 +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) +{ + rt_spin_unlock(lock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) @@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spi= nlock_t *lock, =20 #define spin_trylock(lock) rt_spin_trylock(lock) =20 +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) +{ + return rt_spin_trylock(lock); +} + #define spin_trylock_bh(lock) rt_spin_trylock_bh(lock) =20 #define spin_trylock_irq(lock) rt_spin_trylock(lock) diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index b42d293da38b..83a17eaf5717 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##= _t *lock) \ */ BUILD_LOCK_OPS(spin, raw_spinlock, __acquires); =20 +/* No rwlock_t variants for now, so just build this function by hand */ +static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + for (;;) { + preempt_disable(); + local_interrupt_disable(); + if (likely(do_raw_spin_trylock(lock))) + break; + local_interrupt_enable(); + preempt_enable(); + + arch_spin_relax(&lock->raw_lock); + } +} + #ifndef CONFIG_PREEMPT_RT BUILD_LOCK_OPS(read, rwlock, __acquires_shared); BUILD_LOCK_OPS(write, rwlock, __acquires); @@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlo= ck_t *lock) EXPORT_SYMBOL(_raw_spin_lock_irq); #endif =20 +#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ +noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + __raw_spin_lock_irq_disable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable); +#endif + #ifndef CONFIG_INLINE_SPIN_LOCK_BH noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) { @@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spin= lock_t *lock) EXPORT_SYMBOL(_raw_spin_unlock_irq); #endif =20 +#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ +noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) +{ + __raw_spin_unlock_irq_enable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable); +#endif + #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) { diff --git a/kernel/softirq.c b/kernel/softirq.c index 10af5ed859e7..0c9b2269a8d6 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -9,6 +9,7 @@ =20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt =20 +#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE #include #include #include @@ -88,6 +89,20 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif =20 +DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); + +void _local_interrupt_disable(void) +{ + __local_interrupt_disable(); +} +EXPORT_SYMBOL(_local_interrupt_disable); + +void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +EXPORT_SYMBOL(_local_interrupt_enable); + DEFINE_PER_CPU(unsigned int, nmi_nesting); =20 /* @@ -728,10 +743,19 @@ static inline void __irq_exit_rcu(void) #endif account_hardirq_exit(current); preempt_count_sub(HARDIRQ_OFFSET); - if (!in_interrupt() && local_softirq_pending()) { + /* + * Interrupts may happen between hardirq_disable_enter() and + * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes + * softirq here, we may have a softirq handler calling + * local_interrupt_disable() but it won't disable the IRQ because + * hardirq disabling count is already 1, hence we need to prevent + * invoking softirq when a local_interrupt_disable() is ongoing. + */ + if (!in_interrupt() && !hardirq_disable_count() && + local_softirq_pending()) { /* * If we left hrtimers unarmed, make sure to arm them now, - * before enabling interrupts to run SoftIRQ. + * before enabling interrupts to run softirq. */ hrtimer_rearm_deferred(); invoke_softirq(); --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 364483A963C for ; Fri, 7 Aug 2026 07:02:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086157; cv=none; b=q1MaeOsUCK1gN6yABNqTESEzGcwUD77Wde110Ah43Xk9mUocBoMiFMc2WRkj8EtM7O+0lYSX6rXi2W56P9FS93FwkoZ9OgHAGko5bkxDP8639Ayg0tgJ8dxqIPBI3UUgkuhrGePAwvGLOYE5WtVLbSMpXn/TrhUcpMH5PDGtRjM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086157; c=relaxed/simple; bh=M6+m/0c8IuZ1xZ+UnruHtoqzFvPgWzMV2K9poAY63Gc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ExJEmsqkOy5doyTkuAOjyv42wzTn88hhwp/tZag4IcfZJllDwhMn2NGRbs/TUzFNKsXkGJzMynpAPQhLHdmggB1ztudljeYtKxgMFp/73mYL4+wNGWhzKJ/ZBzdk74eFTLjf18pc/WYxhdheJHdf4hPr74aCgUnX/qt//1UZ9Oc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=fhtS8WTk; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="fhtS8WTk" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 04FCF1F00A3E; Fri, 7 Aug 2026 07:02:34 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086155; bh=+D9X1kYa+VRU9yRO4WbCcrFIABEuEB98nxaJDGa/Hfg=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=fhtS8WTkQfLEhFgeD5UbV1gqUpcGm7Rck9Xg4GbEBUiwXXLjWByeI2p/1vTmnlyk5 rmHecua1jdZ1jLwdz6CDBfjSeS+MlzzOy/3vdPqFBVDLRSI+vQW8qW1QAIQs5mXzPk hyVHTTP29h/DSn7bC8uiBVEOZN5zWUvO8x7A3bvlPJNPe79mQDTpZb7Qq3ZgsHXjMN Y+v0wsEuYkdiUoqXQQsG1HT6UayWsa09wAmAczPJwSsTQ4pJtNBKy55vSIXHJD2psE 91UoJQsCviribBzMfU6CM6I7zr7NZNKNQwpdlG8reNu3PES/b/g+HOOK5qyY/4TekN NvRFAHJngoOyQ== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 1D774F40066; Fri, 7 Aug 2026 03:02:34 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Fri, 07 Aug 2026 03:02:34 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTELT6mPBN+tf+gYDjbmfxF2aJuNia3fHRbo+cksodwgHpv175RY/La8br7LVuLKD+ xpTrulxkYenRUqjuFjoYpWp5kB81fkgSYs4cUOn9Aimg2+sWPDBZPSJD0AkLNVKo9ymnEn XwkNJDHucJ7KY3ALZ5VcWOWe5XHuNtoJV5e7Q54FFSKl7KOd+nrEm1L6dR4Jdu+LjwwV/e gMJ05NqNZsVzzu9M3eDFzAUAJPeoh4s2jJHbhIzGNi7DAZxkNRx1H6hWOvNKjwmDuMbW1g 3/Yvg82sjVAOqgIP9o+xyvh3ddqbO1HSSPQ13n3n5YTt3W/2pxiARls47VLAs4X6PdSfH0 wdndMsJjTG3YJRTfVWddIDNZEXa2Bgy24oJasNFwLCsCmNNPB5bcP4ECzegpNSkN55J9mV 54kuc4ywcJYNiK5PAZKjnDCH8/7Tzihy4t4YMlOJvExlSntHmz1c5abnQTysgytqfLPvr3 F1BM+stAcS5QFoQYc+VO1vaSYyDRN4bdNDI9WPTy92hQW+dDm5dkpo1JrYa7mx2RlkNRNW HSHXJLwSDSd/A3V8AOMBbzzZAJLaC3wP8E9ezecoLNy9FRACV5Elze3eicAW3uWjrqUFfT vZFyihjZvqAVMC9xMdlJiICUo/VsYo/BF8DbeoEjgt0+t5CWKtxDWWYDDdvA X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:33 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 06/18] irq: Add KUnit test for refcounted interrupt enable/disable Date: Fri, 7 Aug 2026 00:02:03 -0700 Message-ID: <20260807070218.27144-7-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul While making changes to the refcounted interrupt patch series, at some point on my local branch I broke something and ended up writing some kunit tests for testing refcounted interrupts as a result. So, let's include these tests now that we have refcounted interrupts. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng --- kernel/irq/Makefile | 1 + kernel/irq/refcount_interrupt_test.c | 109 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 kernel/irq/refcount_interrupt_test.c diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 86a2e5ae08f9..44c4d6fc502a 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) +=3D affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) +=3D debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) +=3D matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) +=3D irq_test.o +obj-$(CONFIG_KUNIT) +=3D refcount_interrupt_test.o diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_int= errupt_test.c new file mode 100644 index 000000000000..ca904dba24b9 --- /dev/null +++ b/kernel/irq/refcount_interrupt_test.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for refcounted interrupt enable/disables. + */ + +#include +#include + +#define TEST_IRQ_ON() KUNIT_EXPECT_FALSE(test, irqs_disabled()) +#define TEST_IRQ_OFF() KUNIT_EXPECT_TRUE(test, irqs_disabled()) + +/* =3D=3D=3D=3D=3D Test cases =3D=3D=3D=3D=3D */ +static void test_single_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); +} + +static void test_nested_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_multiple_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_irq_save(struct kunit *test) +{ + unsigned long flags; + + local_irq_save(flags); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_irq_save(flags); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static struct kunit_case test_cases[] =3D { + KUNIT_CASE(test_single_irq_change), + KUNIT_CASE(test_nested_irq_change), + KUNIT_CASE(test_multiple_irq_change), + KUNIT_CASE(test_irq_save), + {}, +}; + +/* init and exit are the same. */ +static int test_init(struct kunit *test) +{ + TEST_IRQ_ON(); + + return 0; +} + +static void test_exit(struct kunit *test) +{ + TEST_IRQ_ON(); +} + +static struct kunit_suite refcount_interrupt_test_suite =3D { + .name =3D "refcount_interrupt", + .test_cases =3D test_cases, + .init =3D test_init, + .exit =3D test_exit, +}; + +kunit_test_suite(refcount_interrupt_test_suite); +MODULE_AUTHOR("Lyude Paul "); +MODULE_DESCRIPTION("Refcounted interrupt unit test suite"); +MODULE_LICENSE("GPL"); --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id F1AC73ACA5C for ; Fri, 7 Aug 2026 07:02:38 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086160; cv=none; b=sSrmtdoootet5DCGQJJZYwfIakJFCoThua6AbnfeZiVL6/9GxtrL9UbzLma0C+K4Wu8d/koHvI3kn7MUh54U49JPOVGdtymCZpE5TqmM7+CYvhAR3IiFIdKro6aL5x2SoQ2VHOgNlXTtRwgzvXRbgiD/r11fLV4J6LmnWCt9ei0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086160; c=relaxed/simple; bh=hqzDoFZ+FnJ+kwkQ1XUbPH87gH8HIR00iTIZdoqIJfw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=JvkF/ZDwFJZA44cFBjEtqd0sES3cyrzXQyIFQvq4bOAJ3l++hWWQHMdGYYgTVzRHKJP5SON66/6Nv0vtYhSciGto6kqX56yZU4chVPaurLD5Lw322qCnmb/zbBm9viEH2/ftxdfg7KMbaVOmuIxhslsEjjz19i28eM9n3KSijTY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=O1JxGlJB; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="O1JxGlJB" Received: by smtp.kernel.org (Postfix) with ESMTPSA id DEC041F000E9; Fri, 7 Aug 2026 07:02:37 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086158; bh=7uHnXoIwtSGCT9rqIiBE0UwPM81LBEEkAOAS7zp4QXM=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=O1JxGlJBh/hV9iLPY0jY9foQ/tgXlkqmGQEgVOa8+WglAjEqGp0D9pPdaoiBGsdQC oN/6PHI5pgmPmJ5DVJKLqLjQifQhCL3uCAoz1PKxUYtqLVK0YJtzsiOMkMTRzjc+30 d0DDSi8L2a+3fr1VqhlLqE+cNwOHbX0N5UZ/GBr2ZbqWDPkue1LO5DgaSEa4jiO7dO ih6uNiqdTMaAmQVfaiHJzJKPZRyqnO1z3LGrkV7a/B+bG+n7c96vHoO5kv/kPDQcSg Wp+JZKtSG/MtlEqPnhp6WmtCo/U1/UF3oaxWWaiKVOmy8+nhRRI95kOVGo69yvCv3v CU61LTl7czpSw== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 7E2EDF40068; Fri, 7 Aug 2026 03:02:35 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Fri, 07 Aug 2026 03:02:35 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGyvHdDd1GzGWN1VJTYexoeLWbYipFw83SRb2uXMsYl+KFTYaUEpOTqk5K+My00CP NgJOi8LOILsVWSxl4ccyvtrBefkjyfVln7ypRQ+bh49utynffTDp/Bsg/I+mtxl14w83ED 50tRYjk+o0iP8mpu+QvCT+Dk+0ekkSaH2jaUkhQ2IGs7hn31QTDC3HO568nJzP+sCzaxl0 cNxOjeFWag8Xux4s5MnrVTP/LpOs6EV32l4U2+8ipe5mhbLBKcn8VxJg/bq9Q7Ww63Su+/ gyrSa0CaL2q4j6exaTekIVpxNJMTqSgpye9e2bSA0gixFi8opX6xU3WPxpol6ShOOMsGZJ nFqkAbVEMZ00RZpD7c494NV/JTofmnieGnOaMXAmEQp5cypldMKVeKr5aDRiDSkYhlvQHF 7FMWicd4K3e/uHkITRvX43ehpJ+8ZHYuBnJ+epM8QXIGscLq3X2xt9S8chQJRllCiHL/3g KWnXC3QTtIP5WAylRFy/i/nnTEAh1To4p95SiZkU+4mg7sgbe32PzSjvDo6BAcQ6kxVoAy OuduM26mMDNEGxI/h3C1lqWheJp+jnj9NQnmicZWRumTS86LzbGpCY01SqghpygjQJVfVu WFyaZPRjftJlzFaEYaqbUuOJi0HSqmGhQakDI4/IP/napRuNvInPRJHRvOTA X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:35 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 07/18] irq: Add max local_interrupt_disable() nesting level kunit test case Date: Fri, 7 Aug 2026 00:02:04 -0700 Message-ID: <20260807070218.27144-8-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" To confirm the max nesting level of local_interrupt_disable() works, a kunit test is added to the whole test suite. Note that when DEBUG_PREEMPT=3Dy, it'll generate a warning which is expected. Suggested-by: Shrikanth Hegde Signed-off-by: Boqun Feng --- kernel/irq/refcount_interrupt_test.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_int= errupt_test.c index ca904dba24b9..38dfccbaa4d4 100644 --- a/kernel/irq/refcount_interrupt_test.c +++ b/kernel/irq/refcount_interrupt_test.c @@ -52,6 +52,27 @@ static void test_multiple_irq_change(struct kunit *test) TEST_IRQ_ON(); } =20 +static void test_max_nesting_irq_change(struct kunit *test) +{ + for (int i =3D 0; i < __IRQ_MASK(HARDIRQ_DISABLE_BITS); i++) { + local_interrupt_disable(); + TEST_IRQ_OFF(); + } + + + for (int i =3D 0; i < __IRQ_MASK(HARDIRQ_DISABLE_BITS); i++) { + TEST_IRQ_OFF(); + local_interrupt_enable(); + } + + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + static void test_irq_save(struct kunit *test) { unsigned long flags; @@ -79,6 +100,7 @@ static struct kunit_case test_cases[] =3D { KUNIT_CASE(test_single_irq_change), KUNIT_CASE(test_nested_irq_change), KUNIT_CASE(test_multiple_irq_change), + KUNIT_CASE(test_max_nesting_irq_change), KUNIT_CASE(test_irq_save), {}, }; --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 817D03ACA6A for ; Fri, 7 Aug 2026 07:02:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086161; cv=none; b=pjN4M3/qsJLSve7/w2rT6ey/wiPaEUs/7fNzgAmDZs1gufwtcal68UCAQUE8hagufg5bLMqNL+XhGfy5G64/6K9vcu0tyfAqUh7Anjdf89FRDpY9n6V1Y4NiPKPsgfhHanQ6tdTI37jsNxU2PXEDJ4ctJa2yy0x+NDpFsuQzAKQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086161; c=relaxed/simple; bh=ez8mh8KYXsxEZezHI84ye4skuWKbHPcxR5bfxvHXALU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=bICLiOSer+7Zr8/aU2bWFNRm1s8grnN43K2J+pIeP+LKgO7PZC03JpCb1nW0zIuoBauCMoqSaurFPVz5aIKHcBS1jERKHmtm5mh0kUpf52tgtABX4vUimh6vH0Qd2V9lujx8Yphmtn3bdZy+1cuEa7eOqlK2+aef1GL0NZmGK78= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=PVLELrof; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="PVLELrof" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6520D1F00A3F; Fri, 7 Aug 2026 07:02:38 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086159; bh=fJJjzop4LECjkHzK46lASg2zu9hWEBiVBs7rN6nvoKc=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=PVLELrofr9JuwIh7FT4yuYf9f2C62NnX58Lvqm+0dKcNxhTQ3ud5qA/MV/Tvpra7c GM6M2n74lZQDgyNdwQC4Vhhl2Oet1Cfpg7EHTNu1Xp7XFwgVSTfzHIIT1uLNpOSQ5W tJpVzJVw4XnxXmFKOqya7gtODYzazNYpeqlznOPlHLnfs9fhD8Da9FTAdXfXdP0v1a UMiA5tJYUExeB2KYxIIIXgENYYQG+nJC8DYWviOIxehcbpgkQQln5pYaMoAvnSXlmQ JXjevepEKJBNCyZuNXUGeuOFz1Mfg3xk7Yn9Iji2vwi/gVQdjWDPagIrh1bD6wkS0k 18RnYk3pModCg== Received: from phl-compute-08.internal (phl-compute-08.internal [10.202.2.48]) by mailfauth.phl.internal (Postfix) with ESMTP id DD085F40066; Fri, 7 Aug 2026 03:02:36 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-08.internal (MEProxy); Fri, 07 Aug 2026 03:02:36 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGyvHdDd1GzGWN1VJTYexoeLWbYipFw83SRb2uXMsYl+KFTYaUEpOTqk5K+My00CP NgJOi8LOILsVWSxl4ccyvtrBefkjyfVln7ypRQ+bh49utynffTDp/Bsg/I+mtxl14w83ED 50tRYjk+o0iP8mpu+QvCT+Dk+0ekkSaH2jaUkhQ2IGs7hn31QTDC3HO568nJzP+sCzaxl0 cNxOjeFWag8Xux4s5MnrVTP/LpOs6EV32l4U2+8ipe5mhbLBKcn8VxJg/bq9Q7Ww63Su+/ gyrSa0CaL2q4j6exaTekIVpxNJMTqSgpye9e2bSA0gixFi8opX6xU3WPxpol6ShOOMsGci X8KX7wH56UKvKXn9XIdhxlvE2o1o6X0XwFE8nMlN3dxon3F7XcR83EoSu//ZlVLhOTo3Zd ghXsDTTV2Nv8MFGRn4wReX928gEhL88cujsqtqC6KVcD5zVQOf5UWuJjL0dOnXzi8O+OdD 4pKUoiOi73wcOnBEYDKuU4h52ZuIBPet4kBXCWT93blTUJ6gwLrzcKKw1JtOCJM+/Ap5Em cP+OSo5E9LsKKmLCphdtKVLUSvHPjJonq+smso87a2qnyhOSc/ZB/6Q1Jhx517Fq41VGfC j95gU73j5av8LrBwqx4TtqQwqoT7cru8yo2Ooxt2/mvuFil/kiAf04lTYFwA X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:36 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 08/18] locking: Switch to _irq_{disable,enable}() variants in cleanup guards Date: Fri, 7 Aug 2026 00:02:05 -0700 Message-ID: <20260807070218.27144-9-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The semantics of various IRQ disabling guards match what *_irq_{disable,enable}() provide, i.e. the interrupt disabling is properly nested, therefore it's OK to switch to use *_irq_{disable,enable}() primitives. [boqun: Adjust the user-side changes in do_sched_cfs_*_timer() provided by Peter and Lyude] Signed-off-by: Boqun Feng --- include/linux/spinlock.h | 26 ++++++++++++-------------- kernel/sched/fair.c | 12 ++++++------ 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 3d405cc4c121..799a8f7d2741 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -572,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acq= uires(_T), __releases(*(raw #define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(= raw_spinlock_nested, _T) =20 DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t, - raw_spin_lock_irq(_T->lock), - raw_spin_unlock_irq(_T->lock)) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(= raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw= _spinlock_irq, _T) =20 -DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->= lock)) +DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disa= ble(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __release= s(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS= (raw_spinlock_irq_try, _T) =20 @@ -592,14 +592,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acq= uires(_T), __releases(*(raw #define class_raw_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(= raw_spinlock_bh_try, _T) =20 DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t, - raw_spin_lock_irqsave(_T->lock, _T->flags), - raw_spin_unlock_irqrestore(_T->lock, _T->flags), - unsigned long flags) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __release= s(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS= (raw_spinlock_irqsave, _T) =20 DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try, - raw_spin_trylock_irqsave(_T->lock, _T->flags)) + raw_spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __rel= eases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_A= TTRS(raw_spinlock_irqsave_try, _T) =20 @@ -618,13 +617,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_= T), __releases(*(spinlock_t #define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinloc= k_try, _T) =20 DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t, - spin_lock_irq(_T->lock), - spin_unlock_irq(_T->lock)) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spin= lock_t **)_T)) #define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinloc= k_irq, _T) =20 DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try, - spin_trylock_irq(_T->lock)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(= spinlock_t **)_T)) #define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spi= nlock_irq_try, _T) =20 @@ -640,14 +639,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquire= s(_T), __releases(*(spinloc #define class_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spin= lock_bh_try, _T) =20 DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t, - spin_lock_irqsave(_T->lock, _T->flags), - spin_unlock_irqrestore(_T->lock, _T->flags), - unsigned long flags) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(= spinlock_t **)_T)) #define class_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spi= nlock_irqsave, _T) =20 DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try, - spin_trylock_irqsave(_T->lock, _T->flags)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __release= s(*(spinlock_t **)_T)) #define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS= (spinlock_irqsave_try, _T) =20 diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d78467ec6ee1..a46c4ff49f74 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7120,7 +7120,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwid= th *cfs_b) * period the timer is deactivated until scheduling resumes; cfs_b->idle is * used to track this state. */ -static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int over= run, unsigned long flags) +static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int over= run) __must_hold(&cfs_b->lock) { int throttled; @@ -7155,10 +7155,10 @@ static int do_sched_cfs_period_timer(struct cfs_ban= dwidth *cfs_b, int overrun, u * This check is repeated as we release cfs_b->lock while we unthrottle. */ while (throttled && cfs_b->runtime > 0) { - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); + raw_spin_unlock_irq_enable(&cfs_b->lock); /* we can't nest cfs_b->lock while distributing bandwidth */ throttled =3D distribute_cfs_runtime(cfs_b); - raw_spin_lock_irqsave(&cfs_b->lock, flags); + raw_spin_lock_irq_disable(&cfs_b->lock); } =20 /* @@ -7266,7 +7266,7 @@ static __always_inline void return_cfs_rq_runtime(str= uct cfs_rq *cfs_rq) static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) { /* confirm we're still not at a refresh boundary */ - scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) { + scoped_guard(raw_spinlock_irq, &cfs_b->lock) { u64 runtime =3D 0, slice =3D sched_cfs_bandwidth_slice(); =20 cfs_b->slack_started =3D false; @@ -7351,14 +7351,14 @@ static enum hrtimer_restart sched_cfs_period_timer(= struct hrtimer *timer) int idle =3D 0; int count =3D 0; =20 - CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock); + guard(raw_spinlock_irq)(&cfs_b->lock); =20 for (;;) { overrun =3D hrtimer_forward_now(timer, cfs_b->period); if (!overrun) break; =20 - idle =3D do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags); + idle =3D do_sched_cfs_period_timer(cfs_b, overrun); =20 if (++count > 3) { u64 new, old =3D ktime_to_ns(cfs_b->period); --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C838F3ACF17 for ; Fri, 7 Aug 2026 07:02:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086161; cv=none; b=Q9cpqEUkpcnOP3ms8Ic/HPNVUSUiTYGXnF5r4lEJbo/bovDQQuK7vfkCu8RO7g6dsJqqOfSpUPpRUTiFTDargj9t93nLgydyZYdPF+1G1Jt5iR98sY4M2LcpJId2+CKmm+1scKWB6QTN77sD+vK9d4cbmRh6AIW+y0wW3tfATCE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086161; c=relaxed/simple; bh=ec5Ah/8+IE+sh6duXoIfePrYPk/nTB/nNJ9FI66HfV4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=HKv+aHCGgiRynPxjDhob2GngdEvmaqwjy+CuQDmGyoq1+R7Vw51pDzH+t5B8NbCk4hLLic4IN01OJnDgyozaTvfiosJ5yJ7Fqh0LDCe+aA0DQVUjzKr646Upckr3MjJ03YzS+vogTTKzA81+CoYPra4SDKQB3v0beIPjoPFkEMs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=m11ZftOA; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="m11ZftOA" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3165C1F00ACF; Fri, 7 Aug 2026 07:02:39 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086159; bh=0thUqhS5l/CCHvciQUIdxDtNj6hdUCGVYjpithCalyQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=m11ZftOAn7lpulkzJUtOyQQw0ULphlxM8nXGezNFS7Ukf6hNZs2kFO9Q9IwinnDDW /pgRo0r9ePJp5rMBm9V1T2uAcJfLtFHROYvqdlkVQqWGNOpR0yktxFIPzExaaiGODn AsYUv0EyAQPo8dr/jmzljdK/Up2W8C3hb6gMuX0d8AFiv3rnHj5SLIJBchIRqSw/f3 QfypX+zVqluc27U07YDX6CNzf23fHT2OpFSbpjorddbDnlToCZ8hn3V0rRUT3JoNGg cyw9czjIHVblKe+aob3WboyJLu5QOM4Ee8vYLEvGYmpKQx5xyuOeuGruv19ghc94fs B3GlZ/PRM8LuA== Received: from phl-compute-04.internal (phl-compute-04.internal [10.202.2.44]) by mailfauth.phl.internal (Postfix) with ESMTP id 49A89F40069; Fri, 7 Aug 2026 03:02:38 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-04.internal (MEProxy); Fri, 07 Aug 2026 03:02:38 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTELT6mPBN+tf+gYDjbmfxF2aJuNia3fHRbo+cksodwgHpv175RY/La8br7LVuLKD+ xpTrulxkYenRUqjuFjoYpWp5kB81fkgSYs4cUOn9Aimg2+sWPDBZPSJD0AkLNVKo9ymnEn XwkNJDHucJ7KY3ALZ5VcWOWe5XHuNtoJV5e7Q54FFSKl7KOd+nrEm1L6dR4Jdu+LjwwV/e gMJ05NqNZsVzzu9M3eDFzAUAJPeoh4s2jJHbhIzGNi7DAZxkNRx1H6hWOvNKjwmDuMbW1g 3/Yvg82sjVAOqgIP9o+xyvh3ddqbO1HSSPQ13n3n5YTt3W/2pxiARls47VLAs4X6PdSfn7 FjdcdWmyNBOJOpt+tNjN4wi434/K/O/VFxjISq5bGdyaOyy7vv12JndAONN3A+H7z76zu4 eF2Ozq4Tn/Cs1RV4VEGkwDVMevlFYq7W7n9wYWs373dMMWD7n6HKfHglJKDZPLoNcZ63qp 2uA1GH5NSkoE5b/T9rXbH7WpQ/lLTValIdz0VKK0sIIa0FFtqkN21yNMnZ2WXOnb+q+DUc 3Rg38qPk+i0vKaalK55l7fmiZ/uBu1PbWwgIwMFyOqcplC2FmIlLyNe4zWHVKsyMlQ13FZ 1uHocHcpvOhORQ1Cjxv9l/D+s6ZtEZwCGVZovP09fToS8RMQOMZTAExNeK8A X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:37 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 09/18] sched: Remove the unused preempt_offset parameter of __cant_sleep() Date: Fri, 7 Aug 2026 00:02:06 -0700 Message-ID: <20260807070218.27144-10-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The preempt_offset is always 0 in all the callsites of __cant_sleep(), hence remove it. It also allows us to clear up the code a bit by no longer using a "preempt_count() > .." comparison. Signed-off-by: Boqun Feng --- include/linux/kernel.h | 4 ++-- kernel/sched/core.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index e5570a16cbb1..24414c79e59a 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -72,7 +72,7 @@ extern int dynamic_might_resched(void); #ifdef CONFIG_DEBUG_ATOMIC_SLEEP extern void __might_resched(const char *file, int line, unsigned int offse= ts); extern void __might_sleep(const char *file, int line); -extern void __cant_sleep(const char *file, int line, int preempt_offset); +extern void __cant_sleep(const char *file, int line); extern void __cant_migrate(const char *file, int line); =20 /** @@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line); * this macro will print a stack trace if it is executed with preemption e= nabled */ # define cant_sleep() \ - do { __cant_sleep(__FILE__, __LINE__, 0); } while (0) + do { __cant_sleep(__FILE__, __LINE__); } while (0) # define sched_annotate_sleep() (current->task_state_change =3D 0) =20 /** diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..aa116daf21bd 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9199,7 +9199,7 @@ void __might_resched(const char *file, int line, unsi= gned int offsets) } EXPORT_SYMBOL(__might_resched); =20 -void __cant_sleep(const char *file, int line, int preempt_offset) +void __cant_sleep(const char *file, int line) { static unsigned long prev_jiffy; =20 @@ -9209,7 +9209,7 @@ void __cant_sleep(const char *file, int line, int pre= empt_offset) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; =20 - if (preempt_count() > preempt_offset) + if (preempt_count()) return; =20 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AAA513AEB39; Fri, 7 Aug 2026 07:02:41 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086163; cv=none; b=FHgwaVoC57PL36vjV+sTCPP52JKMmpbuiHEgygpTDTtX3Q5BkcHAtmxcNrD7vxIyWFnxmfvx44ZAB1EYvOI5N5DbJqc8jCKiSVjEZ8Y4ETckr4ziduUm2xI95S6e/FXLZheKl+0g9HcqRt2aht3ehKRbr+hcOTmcVaq/qrdr1SY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086163; c=relaxed/simple; bh=gdX2/8yO0N7n17FlM8ctDDZ1lMRo6f/CFk/F/ffOAD4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=XVD1Act8Pc2pgIgApAH1/JKm2esCl5Xd2s6UoJuL5D4u/UDXfeGNRO/LDeNl8BIq8n73ZSnzG3Tp6TqTknFwgJ5YRQjE6lYFU3JHu7FxjqEx993E9XeuaJDa1552Gs58ARq81DixGh7L1Dzo4oCyNjycf+zbM48Ns1ajYS3XIkI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=QXSD/zU9; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="QXSD/zU9" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A51831F00A3E; Fri, 7 Aug 2026 07:02:40 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086161; bh=uaMiBt5vafGBdINwxuR8k+kc9W7/u2yo8z9BluwAE/g=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=QXSD/zU97RwBX3Lrlw7N20fDBja0RoYiJjj7gQN1voUUXR65U01/SZ0uY1LkgYSJp QTjg2Ce2a40AI/nTSdecYgJLNUo+8Av341U4CLq7A/ngyW2hrr0okiDWFsp8VfXz8y CFZPob1PEGbuyjl46dVjO+Dr4cRdBRtuLwvdnP+Ghvf1brJOvU80mROkfKcm3GBlRu dIf+bOavJeLvQAjr610RBVbuh+My6dBkeDkx1ELMz4TfAgGZSSo0zQ8+vUOae9IyJj CK6Aenlq81c1rK7B7d1Mex6xP31pxvP9QzZx40sgKkVCmsAAddLhRnY68KZuq/4Vs0 IqwMyDpLBsjmQ== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id BD546F40066; Fri, 7 Aug 2026 03:02:39 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Fri, 07 Aug 2026 03:02:39 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGyvHdDd1GzGWN1VJTYexoeLWbYipFw83SRb2uXMsYl+KFTYaUEpOTqk5K+My00CP NgJOi8LOILsVWSxl4ccyvtrBefkjyfVln7ypRQ+bh49utynffTDp/Bsg/I+mtxl14w83ED 50tRYjk+o0iP8mpu+QvCT+Dk+0ekkSaH2jaUkhQ2IGs7hn31QTDC3HO568nJzP+sCzaxl0 cNxOjeFWag8Xux4s5MnrVTP/LpOs6EV32l4U2+8ipe5mhbLBKcn8VxJg/bq9Q7Ww63Su+/ gyrSa0CaL2q4j6exaTekIVpxNJMTqSgpye9e2bSA0gixFi8opX6xU3WPxpol6ShOOMsGSJ TIFJHz5UjPAmTocBXDfuX6BnJxjXkyNuTC0AfP877O4xwTlb1iVbYyNH9CcfuQA7lxNjAr IYKBi6vkpSerJnDbHb2jDR7BKiuaaSL4Ue854SiC1TQdz22L8jPTlIRzH3ufjlROAiWH9o jDTsOSv9d6JQw4GfBGaVUlWQn+Aeu52IIGGI4F9QLJx42aYIgoMBT8yKz//peoTcsW+460 OUCZO9QRhIvqIzKqtw199oPBHocIFaOgxzen4mDu3We82/RgxcrawJ2K0vHA2Wg5V6/iCq 1GhCoX9D28EbDjv7zRScpIfWDCS5RStje5XkJ+zWJjYs6gJ75odaLDUsM4+Q X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:39 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 10/18] sched: Avoid signed comparison of preempt_count() in __cant_migrate() Date: Fri, 7 Aug 2026 00:02:07 -0700 Message-ID: <20260807070218.27144-11-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently preempt_count() is always a non-negative int on all archs (PREEMPT_NEED_RESCHED archs will mask out the MSB when returning preempt_count()), hence the checking in __cant_migrate() is in fact just checking whether preempt_count() is 0 or not. In a future change, we are going to use all the 32 bits of preempt_count(), which would make negative int values possible from preempt_count(). Therefore convert the "> 0" comparison into a zero check to prepare for the future change. No functional changes are intended. Signed-off-by: Boqun Feng --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index aa116daf21bd..9b3f1764fa9e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9241,7 +9241,7 @@ void __cant_migrate(const char *file, int line) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; =20 - if (preempt_count() > 0) + if (preempt_count()) return; =20 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A213A3A4F58 for ; Fri, 7 Aug 2026 07:02:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086166; cv=none; b=Q66r1R9frwCan0fk0DnGm8ouMjvh466qWAUtNq6xfDl47d6U2Sn5otrOOqQ6IiX0HI+2ebtd3rI7VzjMDl+ky4iIDaLTGwtuCn/qec6PHCOREHlBhYmd87ED0898NQPFIZpmYnaUm2yDuNET2HOL41xNcFSYa0yBKraoD45uGn4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086166; c=relaxed/simple; bh=nHQ7cZ5+k7RgGGfNoGDfPRnJLZsmOgdHMe6yrMx/xuA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=q8rTtGHk3rf5YN2lKxc2oPELD3nVP9BXjt+ztLnwPhDcG4oRmCHxGRtd1ROujDlhCezaZr3ZaY7Ng/JCuid0y8958+oghFlLZ0ogcWPtLnk4ggGfvBT7cfwTD1pzu/nvseCrmK/6ep1xlEXN2oIu4OEykYQp9q9vKv8KLJ8QTyQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=lPBDzJXG; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="lPBDzJXG" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1F8261F000E9; Fri, 7 Aug 2026 07:02:42 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086162; bh=FDRECyJGIodi8AT12h/ANJTDSaE/Zgze3j4PFvNPEDw=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=lPBDzJXGWNixzUct5sUCkMLZfR8YYtSW52Tin1JbNogG3YeLKIQ1tCOzx1ay9tF38 MIELtvM6ccVncF86F9IkDS15waO1xS1eBtkIzBCaeg/CxiY47UF+q2XXorxUYKdaeA J5WFMiqePZYVpPZynCONTA8CMjvRRcndLxLfaT5lK1gBIysuB422t5febgKt9EHzyr ScKuzi5sr4S9g1Sd6xiMaAVAQ3Qs7itPnFVYPeYG+PAAxxlcTTu67Bud+92NNzkj7o zW+f4lh6cSL5CD9ao5PjWSOnM2Uvub/QIoOULDe706sUN4XuVtQO36PDIcdamBVnRC OHalignZ7hgmA== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id 38E44F40068; Fri, 7 Aug 2026 03:02:41 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-06.internal (MEProxy); Fri, 07 Aug 2026 03:02:41 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTELT6mPBN+tf+gYDjbmfxF2aJuNia3fHRbo+cksodwgHpv175RY/La8br7LVuLKD+ xpTrulxkYenRUqjuFjoYpWp5kB81fkgSYs4cUOn9Aimg2+sWPDBZPSJD0AkLNVKo9ymnEn XwkNJDHucJ7KY3ALZ5VcWOWe5XHuNtoJV5e7Q54FFSKl7KOd+nrEm1L6dR4Jdu+LjwwV/e gMJ05NqNZsVzzu9M3eDFzAUAJPeoh4s2jJHbhIzGNi7DAZxkNRx1H6hWOvNKjwmDuMbW1g 3/Yvg82sjVAOqgIP9o+xyvh3ddqbO1HSSPQ13n3n5YTt3W/2pxiARls47VLAs4X6PdSfHw PZgY/tsWSFXVCQe2XGHjIRwoZsh63uV1397w+Swj2HHef5pe6QRzTSez6WI+Q7Q4z/D95L xhVaUc7ANWbu39Q1RDnoK5Hb0u200JjeIL1Ur0v9pznDgCq3yu0WI74hfXUHI5gWS2xCpg /p8l7AuXBpxoozFqyERY1YmCQxhPtK5Lyvcur1eOAaJGYePJhu3Sn30oMt+0CEH8ZGHc0Q Loz3pmUIkFZ4mrRvxR1jgnNOAzGDPh0W00ZznvrTFg4RbB8TCv6joKG5eHmKXi9p7I0AoF 216sJhfd7K5yHFL/uRZeMUyVczRQ9GjMZFB/j9K92yUh/nk4S/qmCfN/NKRg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:40 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 11/18] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Fri, 7 Aug 2026 00:02:08 -0700 Message-ID: <20260807070218.27144-12-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" With the changes that enable preempt count to track IRQ disabling nesting, we don't have enough bits in 32-bit preempt count implementation, as a result we move NMI nesting bits out of the 32-bit preempt count. However on the architectures that can support 64-bit preempt count implementation, we can keep the NMI nesting bits in the 32-bit preempt count and avoid maintaining NMI nesting bits outside of the same cache line. Therefore HAS_SEPARATE_PREEMPT_RESCHED_BITS is introduced to allow architectures to select this. Note that under this Kconfig, preempt count is maintained in a 64-bit word however preempt_count() still remains as an int because all the effective bits still fit in (previously we mask out NEED_RESCHED bit in preempt_count()). This should make no functional changes for existing preempt_count() users. Enable this for x86_64 along with the introduction of the Kconfig. [boqun: Undo the __preempt_count_{add,sub}() optimization in 32-bit preempt count since it may introduce {over,under}flow] [boqun: Address the feedback from Shrikanth] Originally-by: Peter Zijlstra Signed-off-by: Boqun Feng --- arch/x86/Kconfig | 1 + arch/x86/include/asm/preempt.h | 55 +++++++++++++------ arch/x86/kernel/cpu/common.c | 2 +- include/linux/hardirq.h | 47 +++++++++++----- include/linux/preempt.h | 23 +++++++- kernel/Kconfig.preempt | 4 ++ kernel/sched/core.c | 12 +++- kernel/softirq.c | 6 ++ .../testing/selftests/bpf/bpf_experimental.h | 2 +- 9 files changed, 115 insertions(+), 37 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..6a7067d20a6a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -326,6 +326,7 @@ config X86 select USER_STACKTRACE_SUPPORT select HAVE_ARCH_KCSAN if X86_64 select PROC_PID_ARCH_STATUS if PROC_FS + select HAS_SEPARATE_PREEMPT_RESCHED_BITS if X86_64 && PREEMPT_COUNT select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX select FUNCTION_ALIGNMENT_16B if X86_64 || X86_ALIGNMENT_16 select FUNCTION_ALIGNMENT_4B diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 1220656f3370..022838589d2a 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -7,10 +7,20 @@ =20 #include =20 -DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count); +DECLARE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count); =20 -/* We use the MSB mostly because its available */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* + * We use the MSB for PREEMPT_NEED_RESCHED mostly because it is available. + */ +#define PREEMPT_NEED_RESCHED (~(((unsigned long)-1L) >> 1)) + +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +#define __pc_dec "decq" +#define __pc_op(op, ...) raw_cpu_##op##_8(__VA_ARGS__) +#else +#define __pc_dec "decl" +#define __pc_op(op, ...) raw_cpu_##op##_4(__VA_ARGS__) +#endif =20 /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -24,18 +34,26 @@ DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count); */ static __always_inline int preempt_count(void) { - return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED; + return __pc_op(read, __preempt_count) & ~PREEMPT_NEED_RESCHED; } =20 -static __always_inline void preempt_count_set(int pc) +/* + * unsigned long preempt count parameter works for both 32bit and 64bit ca= ses: + * + * - For 32bit, "int" (the return of preempt_count()) and "unsigned long" = have + * the same size. + * - For 64bit, the effective bits of a preempt count sits in 32bit, and we + * preserve the NEED_RESCHED bit from the old count. + */ +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; =20 - old =3D raw_cpu_read_4(__preempt_count); + old =3D __pc_op(read, __preempt_count); do { new =3D (old & PREEMPT_NEED_RESCHED) | (pc & ~PREEMPT_NEED_RESCHED); - } while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new)); + } while (!__pc_op(try_cmpxchg, __preempt_count, &old, new)); } =20 /* @@ -58,17 +76,17 @@ static __always_inline void preempt_count_set(int pc) =20 static __always_inline void set_preempt_need_resched(void) { - raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED); + __pc_op(and, __preempt_count, ~PREEMPT_NEED_RESCHED); } =20 static __always_inline void clear_preempt_need_resched(void) { - raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED); + __pc_op(or, __preempt_count, PREEMPT_NEED_RESCHED); } =20 static __always_inline bool test_preempt_need_resched(void) { - return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED); + return !(__pc_op(read, __preempt_count) & PREEMPT_NEED_RESCHED); } =20 /* @@ -77,22 +95,22 @@ static __always_inline bool test_preempt_need_resched(v= oid) =20 static __always_inline void __preempt_count_add(int val) { - raw_cpu_add_4(__preempt_count, val); + __pc_op(add, __preempt_count, val); } =20 static __always_inline void __preempt_count_sub(int val) { - raw_cpu_add_4(__preempt_count, -val); + __pc_op(add, __preempt_count, -val); } =20 static __always_inline int __preempt_count_add_return(int val) { - return raw_cpu_add_return_4(__preempt_count, val); + return __pc_op(add_return, __preempt_count, val); } =20 static __always_inline int __preempt_count_sub_return(int val) { - return raw_cpu_add_return_4(__preempt_count, -val); + return __pc_op(add_return, __preempt_count, -val); } =20 /* @@ -102,7 +120,7 @@ static __always_inline int __preempt_count_sub_return(i= nt val) */ static __always_inline bool __preempt_count_dec_and_test(void) { - return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e, + return GEN_UNARY_RMWcc(__pc_dec, __my_cpu_var(__preempt_count), e, __percpu_arg([var])); } =20 @@ -111,7 +129,7 @@ static __always_inline bool __preempt_count_dec_and_tes= t(void) */ static __always_inline bool should_resched(int preempt_offset) { - return unlikely(raw_cpu_read_4(__preempt_count) =3D=3D preempt_offset); + return unlikely(__pc_op(read, __preempt_count) =3D=3D preempt_offset); } =20 #ifdef CONFIG_PREEMPTION @@ -158,4 +176,7 @@ do { \ =20 #endif /* PREEMPTION */ =20 +#undef __pc_op +#undef __pc_dec + #endif /* __ASM_PREEMPT_H */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3df21d26460..73a6d9f6a78e 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2236,7 +2236,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, curren= t_task) =3D &init_task; EXPORT_PER_CPU_SYMBOL(current_task); EXPORT_PER_CPU_SYMBOL(const_current_task); =20 -DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) =3D INIT_PREEMPT_COUNT; +DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) =3D INIT_PREEMPT_= COUNT; EXPORT_PER_CPU_SYMBOL(__preempt_count); =20 DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) =3D TOP_= OF_INIT_STACK; diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 8d4895531a45..860895a7f4e2 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -10,8 +10,6 @@ #include #include =20 -DECLARE_PER_CPU(unsigned int, nmi_nesting); - extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); =20 @@ -94,6 +92,37 @@ void irq_exit_rcu(void); #define arch_nmi_exit() do { } while (0) #endif =20 +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +static __always_inline void __preempt_count_nmi_enter(void) +{ + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); +} + +static __always_inline void __preempt_count_nmi_exit(void) +{ + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); +} +#else +DECLARE_PER_CPU(unsigned int, nmi_nesting); + +#define __preempt_count_nmi_enter() \ + do { \ + __preempt_count_add(HARDIRQ_OFFSET); \ + /* NMI nesting is represented in 4 bits. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ + __this_cpu_inc(nmi_nesting); \ + preempt_count_set(preempt_count() | NMI_MASK); \ + } while (0) + +#define __preempt_count_nmi_exit() \ + do { \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + if (!__this_cpu_dec_return(nmi_nesting)) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ + } while (0) + +#endif + /* * NMI vs Tracing * -------------- @@ -110,18 +139,14 @@ void irq_exit_rcu(void); do { \ lockdep_off(); \ arch_nmi_enter(); \ - /* Maximum NMI nesting is 15. */ \ - BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ - __this_cpu_inc(nmi_nesting); \ - __preempt_count_add(HARDIRQ_OFFSET); \ - preempt_count_set(preempt_count() | NMI_MASK); \ + __preempt_count_nmi_enter(); \ } while (0) =20 #define nmi_enter() \ do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ - ct_nmi_enter(); \ + ct_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ @@ -129,12 +154,8 @@ void irq_exit_rcu(void); =20 #define __nmi_exit() \ do { \ - unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(HARDIRQ_OFFSET); \ - nesting =3D __this_cpu_dec_return(nmi_nesting); \ - if (!nesting) \ - preempt_count_set(preempt_count() & ~NMI_MASK); \ + __preempt_count_nmi_exit(); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 33fc4c814a9f..8299657f0f86 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -34,14 +34,31 @@ * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_DISABLE_MASK: 0x00ff0000 * HARDIRQ_MASK: 0x0f000000 + * + * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=3Dy, PREEMPT_NEED_RESCHED is put= in a + * separate word and that allows 64bit load-store architectures to 'set' + * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric + * modifications used on preempt_count and still load the whole thing + * (single-copy) atomically, without having to resort to full atomic + * operations. + * + * Because of the above, NMI_MASK bits are different depending on + * HAS_SEPARATE_PREEMPT_RESCHED_BITS: + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=3Dn: + * * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=3Dy: + * NMI_MASK: 0xf0000000 + * (PREEMPT_NEED_RESCHED is in a different word) */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 1 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BIT= S)) =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) @@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_= level(void) * preempt_count() is commonly implemented with READ_ONCE(). */ =20 -#define nmi_count() (preempt_count() & NMI_MASK) -#define hardirq_count() (preempt_count() & HARDIRQ_MASK) +#define nmi_count() (preempt_count() & NMI_MASK) +#define hardirq_count() (preempt_count() & HARDIRQ_MASK) #ifdef CONFIG_PREEMPT_RT # define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK) # define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | sof= tirq_count()) diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 88c594c6d7fc..35f546a042b1 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK config PREEMPT_COUNT bool =20 +config HAS_SEPARATE_PREEMPT_RESCHED_BITS + bool + depends on PREEMPT_COUNT && 64BIT + config PREEMPTION bool select PREEMPT_COUNT diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9b3f1764fa9e..6d88343c3bad 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5973,8 +5973,13 @@ void preempt_count_add(int val) #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? + * + * Cannot detect underflow based on the current preempt_count() value + * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes= all 32 + * bits. */ - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) && + DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) return; #endif __preempt_count_add(val); @@ -6006,7 +6011,10 @@ void preempt_count_sub(int val) /* * Underflow? */ - if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) + unsigned int uval =3D val; + unsigned int pc =3D preempt_count(); + + if (DEBUG_LOCKS_WARN_ON(pc - uval > pc)) return; /* * Is the spinlock portion underflowing? diff --git a/kernel/softirq.c b/kernel/softirq.c index 0c9b2269a8d6..7980a4a232f9 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -103,7 +103,13 @@ void _local_interrupt_enable(void) } EXPORT_SYMBOL(_local_interrupt_enable); =20 +#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +/* + * Any 32bit architecture that still cares about performance should + * probably ensure this is near preempt_count. + */ DEFINE_PER_CPU(unsigned int, nmi_nesting); +#endif =20 /* * SOFTIRQ_OFFSET usage: diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing= /selftests/bpf/bpf_experimental.h index 0159a3d365c8..56520d551abd 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -368,7 +368,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup,= const char *name__str, #define SOFTIRQ_BITS 8 #define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 1 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BIT= S)) =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1AD6C3B27DB; Fri, 7 Aug 2026 07:02:44 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086165; cv=none; b=Tb1sOxoJjSeI684nVHoVxjFSQwBPNLijAJv49xjlO53lIjGosLzZuxFQ9NOConwVL27kG+fADAHB9bEd6iijp08NfehnPYyljNIq+EoQ442IWz/bg9+TBK4M+BAeKWB6FUJKOkICMcMc+MDOyW0uFhzQb4xlg3IyZtQ+v8M2u8s= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086165; c=relaxed/simple; bh=JqcH/mibcR84zB0h7wBnVEuD0X2zbft/81pdfe10n0s=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=SkF3BSgHc0NcjZCj3WXicbG6tGJNFoTXWmXbcHGPQ0C0RwAXRf6LXdgykDlJkurC+8s+KFOxuYhffEtqe/Hj3uJi92FkJdS2tnhbk7ufEu1Jw9dqlTNJ6FudcI8cnD1ppAjOGK3fxPAj7S3sJZRw5gRHhoI9jR3KuCd4hilkKbs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ivsHK4Z9; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ivsHK4Z9" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9737B1F00AC4; Fri, 7 Aug 2026 07:02:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086164; bh=3WjTSvvh4sYtO98lql8YoYHvCvWGaUwBwCoDwXD+M1o=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ivsHK4Z9C6tCQcYjd0aALisTfGCNfCctoAru3oB/uKfT21wLo8SUgiyxVg/iPot1j epSVsJiJXiaZISkJ9TxjIOtHVof2UzVA3UmZt3bKlqIzgiBOAqyNg5AC0DRO7vFoGP uYZg99r1LqDiXWrNbjVMY4saJ6MMDSgyts8xaXOyhL6fV67lyv9nDuM6VuKT+6Yjnm wWXVDdhM1rmHdp5JSo73ds7s7i/TXVu1mehK6XD8CJZ17DaDAlr1k3Y/F2gws1nFst GDa1mfDMmksUklpEsB03GcHqwVxMhDsjnEJTUPgDdL/dt4gmDpbEzIaeCPqxn6i/Nw 6bG0G55SgRHbg== Received: from phl-compute-04.internal (phl-compute-04.internal [10.202.2.44]) by mailfauth.phl.internal (Postfix) with ESMTP id ADBF9F40066; Fri, 7 Aug 2026 03:02:42 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-04.internal (MEProxy); Fri, 07 Aug 2026 03:02:42 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTELsVbnxrvhPb2j61weS49pxhnQ59QQ/kkvzVAF7RfvMFBahj3W+J/ZYvUaUDGUKL rkIpWI9Kqh7tdFeNxTw91APFY2yZOuGFzh0R7L9linn/K/VkHskzHhqZIAbhz+FzEI2/39 p/Gf/98yNPd7kqv7ASor+gx/DK4WZv4tGUkwpXImaVpTVn9yjG8Hsrc4m1zuPGR3jPjYfV aGaWr2EDt/5hU/phchzQPFGxTjKWqqv3f39Pht190FqbXj8fyXB6xLi7X5zcPt/Liyw23y b+4UgNbRpkgjQlqdkJa7EiS5FEj8Oxro36Xe5RlI+SlnX1fghWIXGuRondD8d1wH6SlNdT xHO4wfmaw0ezJNYMhR6VbkkU1ajglv6ZQAghdGVVEVMSJJXM8aEuszcfSu8tH3Hc+ljaIs 5bQYjkyvC4nkOcJIH9R5cwGnHgv5onGcUvsX76hhsRdGO5NB8JdZ0xUv0R6X2EBtWQ9MDU PjmG7MuTJ5zYr7OJmUkMIq/Da61KK3s5OtYUNQUhj0q31x3FciuzDIuVMvBlu6aIVaNWZy kAysyC6LVN2MZokAVWQlQA8+Hj+4BIyMOVxNsFv2nUR2EKxqMHDGjoHwT/OW0cG3sRe+nH 4ZBrQBOQZT8SWBVVvfBPIyxcb81iWhtx1st3PHVE9KI6q14tHEdz48wB0UMA X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:42 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 12/18] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Fri, 7 Aug 2026 00:02:09 -0700 Message-ID: <20260807070218.27144-13-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Arm64 already uses 64-bit preempt count and the need reschedule bit is maintained in a separate 32-bit word from the preempt count. Therefore preempt count has enough bits to represent 16 levels of NMI nesting, hence enable it for arm64. This saves a per-CPU variable and additional instructions in the NMI path. Signed-off-by: Boqun Feng --- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..349c3533cd1e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -247,6 +247,7 @@ config ARM64 select PCI_SYSCALL if PCI select POWER_RESET select POWER_SUPPLY + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1AA5E3B71C7; Fri, 7 Aug 2026 07:02:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086167; cv=none; b=KLc6h8UO+fHFsjdMzFL+umfJ3n5ThSFxjNj49wojtWyLry+pgbE9YpS30qvUMIBNHSa2pb4LELsmAAiksNwhnmWQuStStz+dN6D59EDob7cr97L04TpRs9fY9UY0NTDxJ7mSwI5uOBTyS7LLQawdEiMVLzt3UEHg6eoAqkYYvdI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086167; c=relaxed/simple; bh=kLeRiC95t1IZBu689qepS3f7brPx8OxG9N0foKAWud8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=naY2I7raf/h/iMsKp4eSb7X5GdTz89ZBC+stznmeF+7Vg1Gst0mWm7BUSZ7qKWLbIaqy0pOP5O4QYo4JtU5Ot5CSrQe0QP/b4VlPiieWnSrvsshJW072ISZy00WV1NHl06kAaoEdY6gQIXKUzv274fdr2bRg3E9xf6C1UknPFXc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=WB7d6uM8; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="WB7d6uM8" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 140D41F00A3D; Fri, 7 Aug 2026 07:02:44 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086166; bh=fj0vODnnkGxYzHO4JCnC/iMb/acZG7slJd0L06VbQoI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=WB7d6uM8putVq2O74TzVgUM2ZGvr/nnqmCwhX089PFF3q3kpBC/Nm50gzJ2jaiNaS foMQ9yMdj8c5dBtQIRHJ4pHPpDYPYNjNJ3+3AoGtY0eKc9SZQ+ehgE6VMi1FT51l7G VB9/9FIkfDyLjG6jeDzUqRDk3GZ0hdnLhGYC9d1SMMCgodu9wFJCJoefRayZIt6TXY POqan+8ryB09hcmgmI091fix+DX5Zf3DP0B8jID6gVosESa+TusJYxmeuPqVQDl1yD WtQXT8coTVFdSNILVcgNg/dI7eW0SesOJc2XjbLcqIxCF4o2/7VlYcD/08YoadLqHh /tqC8+eXzdCOg== Received: from phl-compute-09.internal (phl-compute-09.internal [10.202.2.49]) by mailfauth.phl.internal (Postfix) with ESMTP id 2B5F1F40067; Fri, 7 Aug 2026 03:02:44 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-09.internal (MEProxy); Fri, 07 Aug 2026 03:02:44 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGymVtxh0AmopRY+TfTfXYPCmloL5eVBz8w4y+3KFrHM+Bwv6o50auh1636bSBYkE H90J+8ZvDwGP+GEnsL03hmghhkCvgcLvZRJlD0Z0K6idzeBe96fVCfVF90zAumgR10DH6A dzrkds6t6yssM5k8WnRKyJY7PoN9yPTJEd8yclKHhKcjgA77IEZ54Ob7IjTR0RbJw1Jd/H MAKXnEz1+HSwhiWHnu3/dGxtHg6i6x+jlrZPffCsJNEQERYySXee/h/1gQg2z905mRcC4Y mB/oQJoiVpI6TFSRaftlL1E+wAZyQH5fXLMWv+6L12SLyEAsyE0kkUaQtsxqspvPUOq9Ew q4mx01rO3hjejbMwbhxCthduMeu3N9aDgo913iixAVTe3XN0h6jyPu6pwjCQAx80rEJOUr 74UR8acqG594NYmFqzMNJ9vyIELj5sl5ikUeQUz7GfTrpV1jB3QnB2Rdb18UcoTG1xLS8o EOqHF9FaXuK39wNUmpPPMUY3zXRmmNGkAVv3+QW8GKuHJg6kdBGTGSrz6dRhFwJ+cw9h4Z wlGenI5YMv301MEMYJD0kqa8bLUiKJc1VRTAmY4kiJNr7WawQ6CUoHkniXaDEH5KQBwsPn 2pwfpjNbmkhVko6vbQxAXiY8VQjT/7m+/frsJRwh8qDoz+IhaDyvVn0WWxeQ X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:43 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" , Heiko Carstens Subject: [PATCH v5 13/18] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Fri, 7 Aug 2026 00:02:10 -0700 Message-ID: <20260807070218.27144-14-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Heiko Carstens Convert s390's preempt_count to 64 bit, and change the preempt primitives accordingly. Signed-off-by: Heiko Carstens [boqun: Apply the corrected comment for asm block] Signed-off-by: Boqun Feng --- arch/s390/Kconfig | 1 + arch/s390/include/asm/lowcore.h | 13 +++++++--- arch/s390/include/asm/preempt.h | 43 +++++++++++++++------------------ 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..378fcd2b6181 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -273,6 +273,7 @@ config S390 select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select PCI_QUIRKS if PCI + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcor= e.h index 3b3ecc647993..5cef215d30e7 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -160,10 +160,15 @@ struct lowcore { /* SMP info area */ __u32 cpu_nr; /* 0x03a0 */ __u32 softirq_pending; /* 0x03a4 */ - __s32 preempt_count; /* 0x03a8 */ - __u32 spinlock_lockval; /* 0x03ac */ - __u32 spinlock_index; /* 0x03b0 */ - __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ + union { + struct { + __u32 need_resched; /* 0x03a8 */ + __u32 count; /* 0x03ac */ + } preempt; + __u64 preempt_count; /* 0x03a8 */ + }; + __u32 spinlock_lockval; /* 0x03b0 */ + __u32 spinlock_index; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ __u8 percpu_register; /* 0x03c0 */ __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preemp= t.h index 0a25d4648b4c..5560d5fca2a3 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -8,11 +8,8 @@ #include #include =20 -/* - * Use MSB so it is possible to read preempt_count with LLGT which - * reads the least significant 31 bits with a single instruction. - */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */ +#define PREEMPT_NEED_RESCHED 0x8000000000000000UL =20 /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -26,25 +23,25 @@ */ static __always_inline int preempt_count(void) { - unsigned long lc_preempt, count; + unsigned long lc_preempt; + int count; =20 - BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) !=3D sizeof(int)= ); - lc_preempt =3D offsetof(struct lowcore, preempt_count); - /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */ + lc_preempt =3D offsetof(struct lowcore, preempt.count); + /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED)= */ asm_inline( - ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n", - "llgt %[count],%[offalt](%%r0)\n", + ALTERNATIVE("ly %[count],%[offzero](%%r0)\n", + "ly %[count],%[offalt](%%r0)\n", ALT_FEATURE(MFEATURE_LOWCORE)) : [count] "=3Dd" (count) : [offzero] "i" (lc_preempt), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS), - "m" (((struct lowcore *)0)->preempt_count)); + "m" (((struct lowcore *)0)->preempt.count)); return count; } =20 -static __always_inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; =20 old =3D READ_ONCE(get_lowcore()->preempt_count); do { @@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc) =20 static __always_inline void set_preempt_need_resched(void) { - __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_cou= nt); } =20 static __always_inline void clear_preempt_need_resched(void) { - __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count= ); } =20 static __always_inline bool test_preempt_need_resched(void) @@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val) =20 lc_preempt =3D offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("asi %[offzero](%%r0),%[val]\n", - "asi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n", + "agsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (val), @@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val) return; } } - __atomic_add(val, &get_lowcore()->preempt_count); + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } =20 static __always_inline void __preempt_count_sub(int val) @@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_t= est(void) =20 lc_preempt =3D offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n", - "alsi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n", + "algsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "=3D@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (-1), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS)); return (cc =3D=3D 0) || (cc =3D=3D 2); #else - return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count); + return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_= count); #endif } =20 @@ -141,7 +138,7 @@ static __always_inline bool should_resched(int preempt_= offset) =20 static __always_inline int __preempt_count_add_return(int val) { - return val + __atomic_add(val, &get_lowcore()->preempt_count); + return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } =20 static __always_inline int __preempt_count_sub_return(int val) --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A97E33BCD24 for ; Fri, 7 Aug 2026 07:02:47 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086170; cv=none; b=LjASPaHIpkgJPOMevwbP+Ltv+LKs08z6POZKO2N5VO/Gfusp4pmAurtjLIc+W0odeQ5IHl54IG41Kjr64VnfYClNsUAQUJQHeylt7KzM0iqRb/F4S8FG8Qfa4AYWcQ+Kui0bESQv0OUiJe2v6Zc6o7Nr4SZEmqBGP0uTzGjiUoU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086170; c=relaxed/simple; bh=PwsYruj68uHekcgGg2+D0S/5Y1JU0FvXwh9yngRw4wU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=M9QODpAm3JR8znJdMy/CxjztV8+ppeZY9SKxd7DCeYTcioYGi1aRy60z1wNQSuOG0XqIuYvi98lyvRU1/QlqyItbLXHM8uSpc3dmULzGjna82QRH5OorPDezVEvAeHcNMEWhEkP3F+pIBRzA1qOdcQqcQrTdjj9xxR2oJCLfYog= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=CVtVisqu; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="CVtVisqu" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 846FF1F00A3A; Fri, 7 Aug 2026 07:02:46 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086167; bh=c1gjxyEZxiWV+lR91hdPXb5LqwKRvifjfGHEayyotg8=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=CVtVisquvWFt7l5b8RIUBlZo4UEbo+VsgUOEIIVCX+bUBnhS5JbaZLJuWYlqOz5ZC MaZApJZhS73GWw6aJO3Es/EyI0LGad2gR9quo+Uy21m6qun6Ri9UAykc9nZCHQJ2ol PgMOS945ns8fUisxT6fU1DSP2X2UgtC5IejtH7fLtl1ymu5lKe260coP4nI9Wlbwcv 9M+0Q26MPd6WM8O0PelenGpT+ez7E7S4H3QBEILGOUSP8cpI6mbUQlhhS7NUxkNn9k e9AuqJMO+E4gYSnQdGXfeu4QU9g/+5+xQBNsonhJwJ3IFqEYuyR/HdCt00CnmJTI9A axjDxSPFxG28g== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 9BCB9F40066; Fri, 7 Aug 2026 03:02:45 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Fri, 07 Aug 2026 03:02:45 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEuIF4BJa02ksz8c6+pbNx+CfKbVvckvqNdWjG0w+J3eOeY4gZPEbHYkHV280IqRE 6CqIVUxK++7rrTQ6KafE3stMxcgHcXTICSkvp/Gm4Pvrjo+EijkyqZYBiiJ3/n6ho2JJoA +z3LEyMMLxnmrvsM8jv7r6QLHuhHe/gVjBCG/U4y8+GBsKMbxk4mVZHM/4KkYYuddOHp5+ c/w7tykyKr75BCb/YMDMzDMDkmcosbnKNvsPeVKdUx+gCCQcP5I1LkxVO7ESqqV2bHBFVp KNlm7eXd5pjuMrlENKGqMcjUt8/3KBTC1wQGBwxnYDR1wKNHQ1uSZR/kSxISILnA/QHkcU XfLd2ul9etVTXhY7dfDAJlITZUnNJOKl7xLWqK6sL2bTy5NQVYVfu6rm7zMbbK+SS87ljI /cwCN8dAzYkJstIRvwSikL0UAm3dhQb3lkuKdUKwh8JOwGlZayK3NP/JtckLdJ0PlRmjpW YjJWGsU8BSdjIh0Mu6ht4o7R25XRUJ6GtP4rQeZj1RaosmRx0SRVMRgtYNWs5s4ticqenh KXABTzF+LBlJ8EJbm6lODXKrsJVzbEeN4xqPsOuh/KS/ugaGmrKqoqZMGQMrLppaci6xe6 j7tEX97a/7KcVUSHK7HsEJq4RSHC2bzHUWn7zm9oZeNYVBu7OJ3DFZu1wAPw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:45 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" , Benno Lossin , Andreas Hindborg Subject: [PATCH v5 14/18] rust: Introduce interrupt module Date: Fri, 7 Aug 2026 00:02:11 -0700 Message-ID: <20260807070218.27144-15-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Lyude Paul This introduces a module for dealing with interrupt-disabled contexts, including the ability to enable and disable interrupts along with the ability to annotate functions as expecting that IRQs are already disabled on the local CPU. Signed-off-by: Lyude Paul Reviewed-by: Benno Lossin Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Signed-off-by: Boqun Feng --- rust/helpers/helpers.c | 1 + rust/helpers/interrupt.c | 18 ++++++++ rust/helpers/sync.c | 5 +++ rust/kernel/interrupt.rs | 89 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 5 files changed, 114 insertions(+) create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/kernel/interrupt.rs diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..0d85b5e68ec2 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -65,6 +65,7 @@ #include "irq.c" #include "fs.c" #include "gpu.c" +#include "interrupt.c" #include "io.c" #include "jump_label.c" #include "kunit.c" diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c new file mode 100644 index 000000000000..51b319bd4c00 --- /dev/null +++ b/rust/helpers/interrupt.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ + local_interrupt_disable(); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ + local_interrupt_enable(); +} + +__rust_helper bool rust_helper_irqs_disabled(void) +{ + return irqs_disabled(); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c index 82d6aff73b04..4f474fe847c4 100644 --- a/rust/helpers/sync.c +++ b/rust/helpers/sync.c @@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(str= uct lock_class_key *k) { lockdep_unregister_key(k); } + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ + lockdep_assert_irqs_disabled(); +} diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs new file mode 100644 index 000000000000..a880ec3b8538 --- /dev/null +++ b/rust/kernel/interrupt.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Interrupt controls +//! +//! This module allows Rust code to annotate areas of code where local pro= cessor interrupts should +//! be disabled, along with actually disabling local processor interrupts. +//! +//! # =E2=9A=A0=EF=B8=8F Warning! =E2=9A=A0=EF=B8=8F +//! +//! The usage of this module can be more complicated than meets the eye, e= specially surrounding +//! [preemptible kernels]. It's recommended to take care when using the fu= nctions and types defined +//! here and familiarize yourself with the various documentation we have b= efore using them, along +//! with the various documents we link to here. +//! +//! # Reading material +//! +//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/52= 0076) +//! +//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/= preempt-locking.html + +use crate::types::NotThreadSafe; + +/// A guard that represents local processor interrupt disablement on preem= ptible kernels. +/// +/// [`LocalInterruptDisabled`] is a guard type that represents that local = processor interrupts have +/// been disabled on a preemptible kernel. +/// +/// Certain functions take an immutable reference of [`LocalInterruptDisab= led`] in order to require +/// that they may only be run in local-interrupt-disabled contexts on pree= mptible kernels. +/// +/// This is a marker type; it has no size, and is simply used as a compile= -time guarantee that local +/// processor interrupts are disabled on preemptible kernels. Note that no= guarantees about the +/// state of interrupts are made by this type on non-preemptible kernels. +/// +/// # Invariants +/// +/// Local processor interrupts are disabled on preemptible kernels for as = long as an object of this +/// type exists. +pub struct LocalInterruptDisabled(NotThreadSafe); + +/// Disable local processor interrupts on a preemptible kernel. +/// +/// This function disables local processor interrupts on a preemptible ker= nel, and returns a +/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible = kernels, this function is +/// a no-op. +/// +/// **Usage of this function is discouraged** unless you are absolutely su= re you know what you are +/// doing, as kernel interfaces for Rust that deal with interrupt state wi= ll typically handle local +/// processor interrupt state management on their own and managing this by= hand is quite error +/// prone. +#[inline] +pub fn local_interrupt_disable() -> LocalInterruptDisabled { + // SAFETY: It's always safe to call `local_interrupt_disable()`. + unsafe { bindings::local_interrupt_disable() }; + + LocalInterruptDisabled(NotThreadSafe) +} + +impl Drop for LocalInterruptDisabled { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariants, a `local_interrupt_disable()` must= be called to create this + // object, hence calling the corresponding `local_interrupt_enable= ()` is safe. + unsafe { bindings::local_interrupt_enable() }; + } +} + +impl LocalInterruptDisabled { + /// Assume that local processor interrupts are disabled on preemptible= kernels. + /// + /// This can be used for annotating code that is known to be run in co= ntexts where local + /// processor interrupts are disabled on preemptible kernels. It makes= no changes to the local + /// interrupt state on its own. + /// + /// # Safety + /// + /// For the whole life `'a`, local interrupts must be disabled on pree= mptible kernels. This + /// could be a context like, for example, an interrupt handler. + #[inline] + pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled { + const ASSUME_DISABLED: &LocalInterruptDisabled =3D &LocalInterrupt= Disabled(NotThreadSafe); + + // Confirm they're actually disabled if lockdep is available + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled(= )`. + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + ASSUME_DISABLED + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9512af7156df..2ee6c24d39c2 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -82,6 +82,7 @@ pub mod impl_flags; pub mod init; pub mod interop; +pub mod interrupt; pub mod io; pub mod ioctl; pub mod iommu; --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id EBF2338AC96; Fri, 7 Aug 2026 07:02:48 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086170; cv=none; b=d/1KU+P//oZXX5ZpudObviqke9OjPGFvOY9MlKtd48MHthwseOaRnJFXq/l7VBGhr4VshwPqcwGa6Hi+ROqGI2oRLrZAMubZdwjws4ewhMKSkxmuADqdo2082thale2+2ImBhjxGYgJwfmA3fwOyQ9/6apVSOBAkCI53aATcnR8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086170; c=relaxed/simple; bh=F7Y7A/nUkO50uzCj7ZdwiNWURz04OlYPwgBgKCWJjsA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=SqQsIPQ3pHLBIj8cr1tQTxwMWhsmprwTWvNUf+rEmzzHX+AaUezGNmcAU6skCLb1NSwUzAzfqci9dmykuQV2/kr37OLQs1pe4SNGrXoWPpeQ5ZBWBkM3jBTmoF/65VviIPSGYd0ZQG+JOemDW9YzKotSb1ba0JNMa/UtojXyMpI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Z5ID1OU6; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Z5ID1OU6" Received: by smtp.kernel.org (Postfix) with ESMTPSA id EE71D1F000E9; Fri, 7 Aug 2026 07:02:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086168; bh=36qEnnu+VHFxkCyChMu3DL672ehrfvpUeGwqcHC+BuQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Z5ID1OU6kP+6S/iAeLO/uJKoZkn7EcBTK2uhUtCL76ZXrViheYcsOS/Kp5UpBH7vp KxOgVRPQoobacMMfN2YYdSQc7YpbKJMUopt+gvS5dcX1CcwiD+3l6GeKUOBzMR44ET KANoRR2i+Tko6mvB7wO6mvAG7ZySdklciGQla7oMi9D9Z8z4pOTEtbeIIl2go/Fj9B TNva/XUBPgdOGBX86rsLHxqAjjiQXL/aX13YOMy7kbCkEAYaSKRbHFwxl3iIVnMCX7 RJyvTGd7arJL/yCts8FR1fuWDgAUvlCWU7vaWrWa/e4axiMDOpBx3KQklo188V328K HwAueFVfoTjSQ== Received: from phl-compute-04.internal (phl-compute-04.internal [10.202.2.44]) by mailfauth.phl.internal (Postfix) with ESMTP id 13B37F40067; Fri, 7 Aug 2026 03:02:47 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-04.internal (MEProxy); Fri, 07 Aug 2026 03:02:47 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGyvHdDd1GzGWN1VJTYexoeLWbYipFw83SRb2uXMsYl+KFTYaUEpOTqk5K+My00CP NgJOi8LOILsVWSxl4ccyvtrBefkjyfVln7ypRQ+bh49utynffTDp/Bsg/I+mtxl14w83ED 50tRYjk+o0iP8mpu+QvCT+Dk+0ekkSaH2jaUkhQ2IGs7hn31QTDC3HO568nJzP+sCzaxl0 cNxOjeFWag8Xux4s5MnrVTP/LpOs6EV32l4U2+8ipe5mhbLBKcn8VxJg/bq9Q7Ww63Su+/ gyrSa0CaL2q4j6exaTekIVpxNJMTqSgpye9e2bSA0gixFi8opX6xU3WPxpol6ShOOMsGrt 6gzwPSi6T7H1uW+B/yrgtkMJ+wD0G7q1QB8MLS7UXfwKHFjnkiGmFisZv08J9LTkNQj90h 8hfDi44+hrzFZSTCLEzh4cXaCFeDccGt36BDaIIdCl+32me0HAa7rIkXqlrctIZDCsLIMu Vp6RoxfvX3X4g1lDNio8Pm7svQTVKh6QM3v8vWDfww9NE7obY+XX3EfZ/Ap7hybAOSAREQ L6111351Vjp0aPkHgUCs7NQqL8u353LQ2PYiSFZVJM7pUixDrcIlVUcwt779VRfr7fGlvF o+1CtZQwCFHg2a/qyhHZn8b8G0AmaTWw4hqvUqWGNxbl201YZMP0BZbAnJdw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:46 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" , Andreas Hindborg Subject: [PATCH v5 15/18] rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers Date: Fri, 7 Aug 2026 00:02:12 -0700 Message-ID: <20260807070218.27144-16-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" spin_lock_irq_disable() and spin_unlock_irq_enable() are inline functions, to use them in Rust helpers are introduced. This is for interrupt disabling lock abstraction in Rust. Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Signed-off-by: Boqun Feng --- rust/helpers/spinlock.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 4d13062cf253..d53400c15022 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinl= ock_t *lock) { lockdep_assert_held(lock); } + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ + spin_lock_irq_disable(lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ + spin_unlock_irq_enable(lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ + return spin_trylock_irq_disable(lock); +} --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 41E7B3BC68E for ; Fri, 7 Aug 2026 07:02:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086171; cv=none; b=kaFDROtGyE+OBH2JuvUV/MVv3SBSItu2a/EG1FFDXT9PhC+Pb3vm/h2Fx9zrEXnyvPBZG0aL4LRf6/hsYg/NGYWwNgJpogvsX2Dqu5pDpKlU9lnbn70wKTSwYImHbXii6mL9yxpUoYmoViRzOHCXaDaZU+RCSWxuYf8WowJe9+Y= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086171; c=relaxed/simple; bh=1B574WZxr85tEXgvJ/lsuCd24hw6r0EvD8jaMRrm6yk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=DOXmFHcWMvCgJzzKYEp3cyV1tSE7ebcjhR0Ht3HVEc8uobOheVKTn8hEC5sgbMhd829z0VzDOP/4thFMJ/kyc6/CTQK4NwTZ3wmA6H9SBTckF6oNrZavJMj6vIt6N73ZVicnNLd1n1ewTbihJPlRFvhTULjKKBvY6oCNS7OpntE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=WfSpFGRN; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="WfSpFGRN" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6647C1F00AC4; Fri, 7 Aug 2026 07:02:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086170; bh=Ehxr5VUxk0jCrAQvRW8oaPBfKD8Ta2YHENyPPAE1YHc=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=WfSpFGRNtOnD7E8Ub4nlQQ2njm2xJPZ9zPzWFKYVikyVnSmChCuI4wfORexmws6bO qS+sM2gxx0bQeVVXR4kqhtLmGOf3IrrWeBRfDYDaEcYOeEvNaeAqNbDGi8SHPWd8H9 JwQZH2ZSuxTOwrxEXWikHnuRmRKuM9rOpIizHMe5v+ifEMOC2l3qSgtwRDExUVpmwy /RVN2DdmWRvEprGm64fvnx5RXYD1mi9WMPNEZ11mifP5Y6WCAOacNM4GraA3O8eegY Hm6EaCKZSBSqIr8xPyj5lj24v6vMNkwT5G0PprD0bspZ7i4TOzLmmhPDx2V1b/Gyoo 9V2X8vdTV7vaQ== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id 7C8D5F40068; Fri, 7 Aug 2026 03:02:48 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-06.internal (MEProxy); Fri, 07 Aug 2026 03:02:48 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTELsVbnxrvhPb2j61weS49pxhnQ59QQ/kkvzVAF7RfvMFBahj3W+J/ZYvUaUDGUKL rkIpWI9Kqh7tdFeNxTw91APFY2yZOuGFzh0R7L9linn/K/VkHskzHhqZIAbhz+FzEI2/39 p/Gf/98yNPd7kqv7ASor+gx/DK4WZv4tGUkwpXImaVpTVn9yjG8Hsrc4m1zuPGR3jPjYfV aGaWr2EDt/5hU/phchzQPFGxTjKWqqv3f39Pht190FqbXj8fyXB6xLi7X5zcPt/Liyw23y b+4UgNbRpkgjQlqdkJa7EiS5FEj8Oxro36Xe5RlI+SlnX1fghWIXGuRondD8d1wH6SlNgk W7+Bu5W2iloERZYb04iIiyBWZiQ9IHDI5GiGJTUIAs4C+7cY9pyfDJJWxdZU3vFcBKJ3FJ 50U+OJeN3bvoL4rA2Mq3y0B+gjfwgHmCGMK/x/Hq5FwEhPZ1o0La/WEYji2pRqa9EfTDC3 kqixZNwr8ct7CXQvvyQI2n++M2vmB/E7x2/thOCUgE+L9zRlKVEBgCGQ1A4Awqoi4e/w5h MS8BNz1qgvc5WDtPOXNxx1gSaD+isdUV6imC2YqlfuX2euAzzZ3v9bckYSH4FH9wA/NEJL 9HNd5Qv1qITMk14/wi+ALzwsuA3Je1QW5qWA1KN9XZJ66Fq5Ijg0nKbQ60sg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:48 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 16/18] rust: sync: Use super::* in spinlock.rs Date: Fri, 7 Aug 2026 00:02:13 -0700 Message-ID: <20260807070218.27144-17-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul No functional changes. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng --- rust/kernel/sync/lock/spinlock.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index ef76fa07ca3a..d75af32218ba 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -3,6 +3,7 @@ //! A kernel spinlock. //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use super::*; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -82,7 +83,7 @@ macro_rules! new_spinlock { /// ``` /// /// [`spinlock_t`]: srctree/include/linux/spinlock.h -pub type SpinLock =3D super::Lock; +pub type SpinLock =3D Lock; =20 /// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend; @@ -91,13 +92,11 @@ macro_rules! new_spinlock { /// /// This is simply a type alias for a [`Guard`] returned from locking a [`= SpinLock`]. It will unlock /// the [`SpinLock`] upon being dropped. -/// -/// [`Guard`]: super::Guard -pub type SpinLockGuard<'a, T> =3D super::Guard<'a, T, SpinLockBackend>; +pub type SpinLockGuard<'a, T> =3D Guard<'a, T, SpinLockBackend>; =20 // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclus= ion. `relock` uses the // default implementation that always calls the same locking method. -unsafe impl super::Backend for SpinLockBackend { +unsafe impl Backend for SpinLockBackend { type State =3D bindings::spinlock_t; type GuardState =3D (); =20 --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 54C7B3955D2 for ; Fri, 7 Aug 2026 07:02:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086173; cv=none; b=Lk1gWQhbuJ7iC9tr9yJbjfzuaVmPuANQeruOeTlCZr6/aZ4hK0vpQyGAqUA2hgPijXFCcSYuBG80RWp12beywZ/D3EaZU1agbpTGGMyOBZp5kON9sAb81BYf/M4SfXLk8qumtW2Hexgw+0C3bJORv7p5JyQ5KHRwzYBrFWgSE7c= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086173; c=relaxed/simple; bh=kZU5uFqdjWiRbrVf1fTNRv9ePx+qKQEq8Hc6Bs6H2Cs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=hepoJhVCpZKmYiXeAv3PQkvQLf+Mr8OZNkbs8kEgATKSjNoCmvYPf2QlvrQvkEVwdcWMO/uU+l8zBWMv6YGNXMACih1QEdBGPZQHoyIt3KE3HYZL1yNwjSnR/QLuPLldTpFurFYgfgOFy/NKKV4Z7yXsA0RV/2aK+nZ9dkk6l4o= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=AOwHgkn3; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="AOwHgkn3" Received: by smtp.kernel.org (Postfix) with ESMTPSA id D0FD91F000E9; Fri, 7 Aug 2026 07:02:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086171; bh=2MYZgIP+osziR2bs888gmmBvgXIfwDVrZfFhcS+axXE=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=AOwHgkn3Zdc2Cjrg1ADruJb0ghMa1PzYZhoUlqK8rF54i6f06pKX/I3+L8dz9TgiO EDa41QMueaS65L6s+gjBdkj5OfclKIu3VYNU2TvmKNtHonSftyWayv+Y6yCppWYoih K0L+Mlh8Vkm2QDJdeXVrGsO1zlOeggbpFr6/i4vlz4ToliqmZ2iYczstrYdwnuy8rw qWhFLidHuEE4Pytc8oCGC+F6/n3CNCwh+7xYrzg1DtnWSKMA5flscKMPN8U5Iz2H3a bH6I0AmqdiyZUlAfYrVpHmxILemaLjf5a8+wbbUdPZBPzEBxYl/WJuY9lFfConZ/Nu C6Pce288JNh5Q== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id E9751F40067; Fri, 7 Aug 2026 03:02:49 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-06.internal (MEProxy); Fri, 07 Aug 2026 03:02:49 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTELsVbnxrvhPb2j61weS49pxhnQ59QQ/kkvzVAF7RfvMFBahj3W+J/ZYvUaUDGUKL rkIpWI9Kqh7tdFeNxTw91APFY2yZOuGFzh0R7L9linn/K/VkHskzHhqZIAbhz+FzEI2/39 p/Gf/98yNPd7kqv7ASor+gx/DK4WZv4tGUkwpXImaVpTVn9yjG8Hsrc4m1zuPGR3jPjYfV aGaWr2EDt/5hU/phchzQPFGxTjKWqqv3f39Pht190FqbXj8fyXB6xLi7X5zcPt/Liyw23y b+4UgNbRpkgjQlqdkJa7EiS5FEj8Oxro36Xe5RlI+SlnX1fghWIXGuRondD8d1wH6SlNBW 5RQjxsaRGjdQy2PmDdfm40Cl5FkLsO02LGHI5fWvMZPCNy/BnMKs83zlmulKgzS45ZHh8Y ui8slDvH0wg10uGffJs7CKt9/PxJgPgd1sG7EcpE2biN48Br+MlRK1WGFQp99CNTRGdMqk A2eVnGdAOzEfWo/iCvmgGRf1s875yzUGh+hsP6FbkLctjRNrJJ6tYOdisyJx6U6Zke11ad euXHrQwtYTWTyW3c+75B/0Iku4x3RLYLMu/BoQwdmlnZl4eO0qUqqUO2oGHZTvNQGI09xk JMok34eac9v7wSw7eaLXJOZjDWC+NF9BrkhSxSK07DlNwjlhJCWlOXAsZk/g X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:49 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 17/18] rust: sync: Add SpinLockIrq Date: Fri, 7 Aug 2026 00:02:14 -0700 Message-ID: <20260807070218.27144-18-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul A variant of `SpinLock` that ensures interrupts are disabled in the critical section. `lock()` will ensure that either interrupts are already disabled or disable them. `unlock()` will reverse the respective operation. [Boqun: Port to use spin_lock_irq_disable() and spin_unlock_irq_enable()] Signed-off-by: Lyude Paul Reviewed-by: Gary Guo Signed-off-by: Boqun Feng --- rust/kernel/sync.rs | 9 +- rust/kernel/sync/lock/global.rs | 3 + rust/kernel/sync/lock/spinlock.rs | 230 ++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..df4f2604ff9b 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -27,7 +27,14 @@ pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBac= kend, GlobalLockedBy}; pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; -pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, + SpinLock, + SpinLockGuard, + SpinLockIrq, + SpinLockIrqGuard, // +}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global= .rs index ec2dd84316fc..ebb10521d8bd 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -306,4 +306,7 @@ macro_rules! global_lock_inner { (backend SpinLock) =3D> { $crate::sync::lock::spinlock::SpinLockBackend }; + (backend SpinLockIrq) =3D> { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index d75af32218ba..872544948e5d 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,6 +4,7 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. use super::*; +use crate::prelude::*; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -143,3 +144,232 @@ unsafe fn assert_is_held(ptr: *mut Self::State) { unsafe { bindings::spin_assert_is_held(ptr) } } } + +/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-= created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on = the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock_irq { + ($inner:expr $(, $name:literal)? $(,)?) =3D> { + $crate::sync::SpinLockIrq::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock= _class!()) + }; +} +pub use new_spinlock_irq; + +/// A variant of `SpinLock` that ensures interrupts are disabled in the cr= itical section. +/// +/// For more info on spinlocks, see [`SpinLock`]. For more information on = interrupts, +/// [see the interrupt module](kernel::interrupt). +/// +/// # Examples +/// +/// The following example shows how to declare, allocate initialise and ac= cess a struct (`Example`) +/// that contains an inner struct (`Inner`) that is protected by a spinloc= k that requires local +/// processor interrupts to be disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// c: SpinLockIrq, +/// #[pin] +/// d: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }), +/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example` +/// let e =3D KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// +/// // Accessing an `Example` from a context where interrupts may not be d= isabled already. +/// let c_guard =3D e.c.lock(); // interrupts are disabled now, +1 interru= pt disable refcount +/// let d_guard =3D e.d.lock(); // no interrupt state change, +1 interrupt= disable refcount +/// +/// assert_eq!(c_guard.a, 0); +/// assert_eq!(c_guard.b, 10); +/// assert_eq!(d_guard.a, 20); +/// assert_eq!(d_guard.b, 30); +/// +/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just = yet, since d_guard is +/// // still in scope. +/// drop(d_guard); // Last interrupt disable reference dropped here, so in= terrupts are re-enabled +/// // now +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrq =3D super::Lock; + +/// A kernel `spinlock_t` lock backend that can only be acquired in interr= upt disabled contexts. +pub struct SpinLockIrqBackend; + +/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`= SpinLockIrq`] using +/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local= processor's interrupt +/// disablement refcount upon being dropped. +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrqGuard<'a, T> =3D Guard<'a, T, SpinLockIrqBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclus= ion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockIrqBackend { + type State =3D bindings::spinlock_t; + type GuardState =3D (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for = writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `p= tr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock_irq_disable(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardStat= e) { + // SAFETY: The safety requirements of this function ensure that `p= tr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock_irq_enable(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initial= ized before use. + let result =3D unsafe { bindings::spin_trylock_irq_disable(ptr) }; + + if result !=3D 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initial= ized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +#[kunit_tests(rust_spinlock_irq_condvar)] +mod tests { + use super::*; + use crate::{ + sync::*, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, + }; + + struct TestState { + value: u32, + waiter_ready: bool, + } + + #[pin_data] + struct Test { + #[pin] + state: SpinLockIrq, + + #[pin] + state_changed: CondVar, + + #[pin] + waiter_state_changed: CondVar, + + #[pin] + wait_work: Work, + } + + impl_has_work! { + impl HasWork for Test { self.wait_work } + } + + impl Test { + pub(crate) fn new() -> Result> { + Arc::try_pin_init( + try_pin_init!( + Self { + state <- new_spinlock_irq!(TestState { + value: 1, + waiter_ready: false + }), + state_changed <- new_condvar!(), + waiter_state_changed <- new_condvar!(), + wait_work <- new_work!("IrqCondvarTest::wait_work") + } + ), + GFP_KERNEL, + ) + } + } + + impl WorkItem for Test { + type Pointer =3D Arc; + + fn run(this: Arc) { + // Wait for the test to be ready to wait for us + let mut state =3D this.state.lock(); + + // Make sure the interrupts actually turned off + // SAFETY: It's always safe to call `lockdep_assert_irqs_disab= led()` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + while !state.waiter_ready { + this.waiter_state_changed.wait(&mut state); + } + + // Deliver the exciting value update our test has been waiting= for + state.value +=3D 1; + this.state_changed.notify_sync(); + } + } + + #[test] + fn spinlock_irq_condvar() -> Result { + let testdata =3D Test::new()?; + + let _ =3D workqueue::system().enqueue(testdata.clone()); + + // Let the updater know when we're ready to wait + let mut state =3D testdata.state.lock(); + state.waiter_ready =3D true; + testdata.waiter_state_changed.notify_sync(); + + // Wait for the exciting value update + testdata.state_changed.wait(&mut state); + assert_eq!(state.value, 2); + Ok(()) + } +} --=20 2.50.1 (Apple Git-155) From nobody Tue Sep 29 13:19:12 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 18A88453A56; Fri, 7 Aug 2026 07:02:54 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086175; cv=none; b=nJZo2j3Hbgx5652TxmHhF7AysvWfhRBkjvnDhJvWtT/C1xOnZoGM8RAlUCw80E+WTBg+47+ZfvtLuYh6d3Kfq4Fy5dQhJBBh1g+URa2OOujLcyZCAzwUuOATIIvVE0RlDh6EwMgeVrQcAXUfkLEhUsSq9BiiazWTNpYmGaUL720= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1786086175; c=relaxed/simple; bh=yOW+vEyMmlLd/rvdAQOpfI7rRLssPojgy8OC8Xdb7b8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=HRYek8cJmLzK3lCqyS6BpXzo6eWyTMFA/xkXPGAVo6XGX+CswgrwknX7vP6b/L9Xrz8+BkzwshiMzrOTs51SrjQJCvfE1RbP7Greji5+0HVENYEHkLfqxIEc4FYYQz3bm1HnAB3YfQSju8tD+io2ft4DKCafkD9iqihLD5xiw+w= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=E667DoEu; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="E667DoEu" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 520331F00A3E; Fri, 7 Aug 2026 07:02:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1786086174; bh=J5r9F5NVhPBA6Gj8qaTBPl3yuPTDh/upo+Mtrw3vN6s=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=E667DoEuUg5FLa1rShvLUT1sgtX8nZrv0/CNyUHS9tjTpKR3AHVE0rGTZU2faQXTR wxwZofYFNWikHwGFcwZFOd1QUjBFEReeGlmbvhtsYVr0VjqJlUDxhvcdN1EsHgWw9I jcHvBW6vipNO9bt7u33iNmx4suVEh6Npaq0tJ4v9Qh98LoJp/xnWr+3kJtV6gpx4bI 69N90nLuiTPyigHV8xQW4VXRmCW649sl3yzH3Kt6t6JosrcsyTNapfhzWb5bcBc+Hr SgBBClSi1ugk5Fbl4GJKvfWbqXQ10amdKSA32DS+TmPw0jiJ7aF10ufYAtefOuhUyl lvDYsX2eDAGdw== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 67C2DF40067; Fri, 7 Aug 2026 03:02:52 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Fri, 07 Aug 2026 03:02:52 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTE8lydczuBK1/wbIQK6YmAUX4f+Ec5X1qeyT1wkaZMDCCrXPb/HwP7HtUh+NFCVhL clfOjleQYAT5amWLr98/fJKXH/Ntk/RmBOsiX4zJwLbn9uvLVg0Z2RCdCAyc3px4IZj1/F ZXlNyt+DCzc16VlaYTrYu1UhJyVcL7yhJjjdgA1mmIOIu4WyN8hdfZ77TUSFvr7zainkfb /D4Vb+s510TJTP/ca9Uc2yFXK2vJOFV/1XWJhRwp5UtJWnttiGxniEnW/HP1Cjlu1j23Af wuD2iIZFmRtkrayImScufIH/fO2ZEneI3YN/an93v1hbc/Xl2wv2f6L6jtPknYrY05temU A4AyfVg5tI3rD7OfZHOqnLTfrnA8qYbt1WUCAaEWNw3uNyiqjssGxytnvcZEcG3Wxj1RmG EpEmNJpld7TnmkxXF0FrfvaIHEILdREMWDo/tFvMDtZ5QPSUVxB7utLfnilcV1rsgMiKC1 qKFMAAIB6ZbGT+ttHZ6w0pNFLHbRrCUyJT9brHBy27MoX9uvyVYb1Jo4V+qC8WtbnxnpGJ yaEY29CuWL6Vt0pTYHbdqyYTQpa+zzMnOdPBk4tVblMk8zOKSkjQIuG03dH1wHHgjE98kR rPAGFfTx/X8lm6htFLxcPjzxYaTm1NV03d6C3tO7V1Oj/3n8zL7S8GOJYmJw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Fri, 7 Aug 2026 03:02:51 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Shrikanth Hegde , Madhavan Srinivasan , "Christophe Leroy (CS GROUP)" Subject: [PATCH v5 18/18] rust: sync: Introduce SpinLockIrq::lock_with() and friends Date: Fri, 7 Aug 2026 00:02:15 -0700 Message-ID: <20260807070218.27144-19-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260807070218.27144-1-boqun@kernel.org> References: <20260807070218.27144-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul `SpinLockIrq` and `SpinLock` use the exact same underlying C structure, with the only real difference being that the former uses the irq_disable() and irq_enable() variants for locking/unlocking. These variants can introduce some minor overhead in contexts where we already know that local processor interrupts are disabled, and as such we want a way to be able to skip modifying processor interrupt state in said contexts in order to avoid some overhead - just like the current C API allows us to do. In order to do this, we add some special functions for SpinLockIrq: lock_with() and try_lock_with(), which allow acquiring the lock without changing the interrupt state - as long as the caller can provide a LocalInterruptDisabled reference to prove that local processor interrupts have been disabled. In some hacked-together benchmarks we ran, most of the time this did actually seem to lead to a noticeable difference in overhead: From an aarch64 VM running on a MacBook M4: lock() when irq is disabled, 100 times cost Delta { nanos: 500 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 292 } lock() when irq is enabled, 100 times cost Delta { nanos: 834 } lock() when irq is disabled, 100 times cost Delta { nanos: 459 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 291 } lock() when irq is enabled, 100 times cost Delta { nanos: 709 } From an x86_64 VM (qemu/kvm) running on a i7-13700H lock() when irq is disabled, 100 times cost Delta { nanos: 1002 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 729 } lock() when irq is enabled, 100 times cost Delta { nanos: 1516 } lock() when irq is disabled, 100 times cost Delta { nanos: 754 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 966 } lock() when irq is enabled, 100 times cost Delta { nanos: 1227 } (note that there were some runs on x86_64 where lock() on irq disabled vs. lock_with() on irq disabled had equivalent benchmarks, but it very much appeared to be a minority of test runs.) While it's not clear how this affects real-world workloads yet, let's add this for the time being so we can find out. This makes it so that a `SpinLockIrq` will work like a `SpinLock` if interrupts are disabled. So a function: (&'a SpinLockIrq, &'a LocalInterruptDisabled) -> Guard<'a, .., Spin= LockBackend> makes sense. Note that due to `Guard` and `LocalInterruptDisabled` having the same lifetime, interrupts cannot be enabled while the Guard exists. Signed-off-by: Lyude Paul Reviewed-by: Gary Guo Signed-off-by: Boqun Feng --- rust/kernel/sync/lock/spinlock.rs | 92 ++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index 872544948e5d..aafc80125f59 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,7 +4,10 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. use super::*; -use crate::prelude::*; +use crate::{ + interrupt::LocalInterruptDisabled, + prelude::*, // +}; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -160,6 +163,16 @@ macro_rules! new_spinlock_irq { =20 /// A variant of `SpinLock` that ensures interrupts are disabled in the cr= itical section. /// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bind= ings will modify the +/// interrupt state to ensure that local processor interrupts remain dis= abled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`]= token is present and +/// local processor interrupts are already known to be disabled, in whic= h case the local +/// interrupt state will not be touched. This method should be preferred= if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// /// For more info on spinlocks, see [`SpinLock`]. For more information on = interrupts, /// [see the interrupt module](kernel::interrupt). /// @@ -213,7 +226,47 @@ macro_rules! new_spinlock_irq { /// # Ok::<(), Error>(()) /// ``` /// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_w= ith()`] in a function +/// which can only be called when local processor interrupts are already d= isabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in n= o-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled= ) { +/// // Because we know interrupts are disabled from interrupt_disable,= we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e =3D KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard =3D local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// /// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with pub type SpinLockIrq =3D super::Lock; =20 /// A kernel `spinlock_t` lock backend that can only be acquired in interr= upt disabled contexts. @@ -278,6 +331,43 @@ unsafe fn assert_is_held(ptr: *mut Self::State) { } } =20 +impl Lock { + /// Casts the lock as a `Lock`. + #[inline] + fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisa= bled) -> &'a SpinLock { + // SAFETY: + // - `Lock` and `Lock` = both have identical data + // layouts. + // - As long as local interrupts are disabled (which is proven to = be true by _context), it + // is safe to treat a lock with SpinLockIrqBackend as a SpinLock= Backend lock. + unsafe { core::mem::transmute(self) } + } + + /// Acquires the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock= ::lock()`] function when + /// possible for [`SpinLockIrq`] locks. + #[inline] + pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) ->= SpinLockGuard<'a, T> { + self.as_lock_in_interrupt(context).lock() + } + + /// Tries to acquire the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock= ::try_lock()`] function + /// when possible for [`SpinLockIrq`] locks. + /// + /// Returns a guard that can be used to access the data protected by t= he lock if successful. + #[must_use =3D "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock_with<'a>( + &'a self, + context: &'a LocalInterruptDisabled, + ) -> Option> { + self.as_lock_in_interrupt(context).try_lock() + } +} + #[kunit_tests(rust_spinlock_irq_condvar)] mod tests { use super::*; --=20 2.50.1 (Apple Git-155)