From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7096C472543; Tue, 4 Aug 2026 16:14:59 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860105; cv=none; b=odttlONdUWXgS7PR4f0Lb0Exq4KFsO5A+036KQ+caFv7GuBZGTXsnKtGdNebvTB+6gw9SSRLqVADc+7sTFGUv7Bs9r3T7Dren3vciCd+KnsKdGwS8vmAFlV9Ic4zBEDNrMGk4vjcWBeyfNgKAe23HVgNG8daCiiSu28bkfXGVsY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860105; c=relaxed/simple; bh=6ESVuJjZ9ui3/4DwNjKV/PraMIkzSH0Vd92IDm2WfD8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=FGE8u7+ApkMXyACXYPeWJWif8MFm4wgeDuzcn9BN3lCfKgthb1v1xJPzDfsUkgG51u//E9anfk9CAuAp35B36n1SkkJ+z53dzMKilEVJM6Fly3QjgJ8A/ykpxiw/x570ZUyAFJ7jvITt0Thy2CBUQWPfSlbIWSEa6UynMI0EJnE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=esgkY0E5; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="esgkY0E5" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B41881F00A3F; Tue, 4 Aug 2026 16:14:57 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860098; bh=3oQWJsZRvEwPxCRQ6qISEKyc40r/iISGRFtA3wGNP7M=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=esgkY0E59U9F3yWySHsx5Y/eZVguUjSKQJYcl1OMWUFiQd+MEWsnxNsw2BdJh2AvK EgIM4sW27tDcCciC+XOEykR1xjTnbE1iLcbhkeurYDfECDOWQ8ChX6O0g+AhsBMkKl aVzv8CN4lgFRU8+Kv3Y+wR96Awx0IbRKdhxUa+4S+uKsL4sCA8ThsxtcGT0+S4hBsQ kruKvNwbsZa/GILRxFs8M5K3F63S4OyS5TD0o3e0SiDFPh8I2cZiPXJmGIbvEY6aS+ Fjne69qhQYZdJx3K5fbCYhQG/Y4Rj96AGSOwQfc2E036uEkX9z9uyyUlVRkyVDP2WQ eFprVfCV1OnJw== Received: from phl-compute-03.internal (phl-compute-03.internal [10.202.2.43]) by mailfauth.phl.internal (Postfix) with ESMTP id E7497F40067; Tue, 4 Aug 2026 12:14:56 -0400 (EDT) Received: from phl-frontend-04 ([10.202.2.163]) by phl-compute-03.internal (MEProxy); Tue, 04 Aug 2026 12:14:56 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbeX OOMahMIm/rhqoL6GY9fUxxQCI3Bldg4POHtkDT2cA3TduIwl8MK74pMhGV8iQEg2pGCmNA dAjUEvTz4fQkubJvXGnKSf1mCzRi0Ci9XwBw36OJS3H/BPlhLDp738aj7497fBQmHtEWSF keXMoIg33iJHc+mNBqrj7/HlSOJdbkpJ2w08tZ9wbZvv564XLx/G4N2BVXcIrJNyZoOJQD 4MzhRAyje4qgb6BjVttB/5fTLsQw1tqfnLc/Bo4mMQvWcaxbbr7l1DGrCH+dpUJSaWYnoB GfX/2pkWszUtVSXzx0agoO5siO3/s7PspXuaIbjVEKxJbUbviICBUUTt54pg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:14:56 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Joel Fernandes Subject: [PATCH v4 01/17] preempt: Track NMI nesting to separate per-CPU counter Date: Tue, 4 Aug 2026 09:14:23 -0700 Message-ID: <20260804161447.84806-2-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Joel Fernandes Move NMI nesting tracking from the preempt_count bits to a separate per-CPU counter (nmi_nesting). This is to free up the NMI bits in the preempt_count, allowing those bits to be repurposed for other uses. Reduce NMI_BITS from 4 to 1, using it only to detect if we're in an NMI. The per-CPU counter currently caps nesting at 15. [boqun: Address Steven Rostedt's comment on the BUG_ON() condition] [boqun: Use preempt_count_set() in __nmi_exit() to avoid underflow] Suggested-by: Boqun Feng Signed-off-by: Joel Fernandes Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng --- include/linux/hardirq.h | 17 +++++++++++++---- include/linux/preempt.h | 9 +++++++-- kernel/softirq.c | 2 ++ tools/testing/selftests/bpf/bpf_experimental.h | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index d57cab4d4c06..8d4895531a45 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -10,6 +10,8 @@ #include #include =20 +DECLARE_PER_CPU(unsigned int, nmi_nesting); + extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); =20 @@ -102,14 +104,17 @@ void irq_exit_rcu(void); */ =20 /* - * nmi_enter() can nest up to 15 times; see NMI_BITS. + * nmi_enter() can nest - nesting is tracked in a per-CPU counter. */ #define __nmi_enter() \ do { \ lockdep_off(); \ arch_nmi_enter(); \ - BUG_ON(in_nmi() =3D=3D NMI_MASK); \ - __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ + __this_cpu_inc(nmi_nesting); \ + __preempt_count_add(HARDIRQ_OFFSET); \ + preempt_count_set(preempt_count() | NMI_MASK); \ } while (0) =20 #define nmi_enter() \ @@ -124,8 +129,12 @@ void irq_exit_rcu(void); =20 #define __nmi_exit() \ do { \ + unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + nesting =3D __this_cpu_dec_return(nmi_nesting); \ + if (!nesting) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index d964f965c8ff..586f96688325 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,6 +17,8 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) + * - bits 16-19 are the hardirq count (max # of hardirqs: 16) + * - bit 20 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -24,16 +26,19 @@ * there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * + * NMI nesting depth is tracked in a separate per-CPU variable + * (nmi_nesting) to save bits in preempt_count. + * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00f00000 + * NMI_MASK: 0x00100000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) diff --git a/kernel/softirq.c b/kernel/softirq.c index 4425d8dce44b..10af5ed859e7 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -88,6 +88,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif =20 +DEFINE_PER_CPU(unsigned int, nmi_nesting); + /* * SOFTIRQ_OFFSET usage: * diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing= /selftests/bpf/bpf_experimental.h index 67ff7882299e..e4e12001fce9 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -367,7 +367,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup,= const char *name__str, #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6AA5447042C for ; Tue, 4 Aug 2026 16:15:03 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860107; cv=none; b=aa7jCSPhvRrFdRSBVLPQt7L5AaYInrY25y5XkjizqMHk45KsgDHj+vOtvxiIjfnr+g1hE1YR3DqGYnJt3bpnlh5sP1wp5WXV7MwbAk7FddIv8IXAWp39kQkQ7/Cx9cRHuFHMipSBW4bHoJH8ycTmTmuxPF1ZHVX4h8Yx1bro4uI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860107; c=relaxed/simple; bh=TcirFITXiiVEzgBfiZWswwyD92L/VOu9Yzk80E8ZDYs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=dxlvlOoF6dtAJMaSWtO5VT7niaRGTvANeO0cBEmckVBkB0rNa6izGPq/3xQcAWOe0TdLGDUMblsmM9DnmniYpZqabhyGfCpVnzOtX79W4WDRlw6ZATYWNi6Zv/EJfzc7Yikz53qPCEqx6CFcH32eZS7qj9p2LthQuIeyAUIkfOQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=UyyN90YI; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="UyyN90YI" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3A49B1F00A3A; Tue, 4 Aug 2026 16:14:59 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860099; bh=koUnNVxTgcqVkfNjlw+37XcYk2uWFTpcvzbP8xX2P9Q=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=UyyN90YImr5wPHR578JLTn+kOt+3zdGIGnD4livnjMSQVotMo29MfTVhjPJw8SqGy UoQldomol5dlq9x0eTVx/tg70dNKnQKKEl4g/zNKypDWVQx3SzoWLsHcE03Pa2U37K K6FmsPBKfodMCwGxZwYVYEaEkB7s1uqxqNVamlmFNzw/udECCeAsx5ramnsE+HVBOf prSFFO9Pujtvy0pmt1Fqxm1mXSnJfGtk6NQz5fzSYrPzzYykEjAHNbjRZqfD9h1uWa MGsvAT+sXKkxK3tq7xkLWxOBW960nAKhWD7Owgb3WbXD1WBiQ3H8IlMYdT6PTFbkbv zH/ak0AicyA5w== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 7963EF40066; Tue, 4 Aug 2026 12:14:58 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Tue, 04 Aug 2026 12:14:58 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbVO aenDcH08zAIddQAApNItbNBUIw1z7JHORMAyK4l8wvX9/ThX52etRbCZ39xlVgW88vQTdA X1xAqTDV25os7op2jSYrSeeqVssydWsoUNNqFU3p9mymoBS743aZ6ByDXbqoKrFJzxvU1j hzdh2Tr1Qqc8DHk/dYXgSfw7b3du3kRnCeyoL7iz/CAMr8J/GARun6ksdy3Tuiq8p+mo/J Q5GUx3pre9uwz8pgPcVxnG7tffuVua8jufsRJBgS+8sdUY1U26smaR9Ja2jkDehhxdA6o3 w3MU78u1ctOmqWRtsnFpO8b4BVHClOND4i7+1GXN8wzerf540bsLEPQAHgVw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:14:57 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 02/17] preempt: Introduce HARDIRQ_DISABLE_BITS Date: Tue, 4 Aug 2026 09:14:24 -0700 Message-ID: <20260804161447.84806-3-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In order to support preempt_disable()-like interrupt disabling, that is, using part of preempt_count() to track interrupt disabling nesting level, change the preempt_count() layout to contain 8-bit HARDIRQ_DISABLE count. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng --- include/linux/preempt.h | 16 +++++++++++----- tools/testing/selftests/bpf/bpf_experimental.h | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 586f96688325..e2d3079d3f5f 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,8 +17,9 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) - * - bits 16-19 are the hardirq count (max # of hardirqs: 16) - * - bit 20 is the NMI flag (no nesting count, tracked separately) + * - bits 16-23 are the hardirq disable count (max # of hardirq disable: 2= 56) + * - bits 24-27 are the hardirq count (max # of hardirqs: 16) + * - bit 28 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -31,29 +32,34 @@ * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 - * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00100000 + * HARDIRQ_DISABLE_MASK: 0x00ff0000 + * HARDIRQ_MASK: 0x0f000000 + * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 #define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) =20 #define __IRQ_MASK(x) ((1UL << (x))-1) =20 #define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_= DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) =20 #define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT) #define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) #define NMI_OFFSET (1UL << NMI_SHIFT) =20 diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing= /selftests/bpf/bpf_experimental.h index e4e12001fce9..0159a3d365c8 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -366,17 +366,20 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgrou= p, const char *name__str, =20 #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 #define NMI_BITS 1 =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) =20 #define __IRQ_MASK(x) ((1UL << (x))-1) =20 #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_= DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) =20 --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E01A8474278 for ; Tue, 4 Aug 2026 16:15:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860110; cv=none; b=uoNmu6FCq/Ze88k+0W/qu53Ihck/xDeZEgPZhMjit3avGPgtv7pSS9XXodRWOcU4JI2u1ljU6BP5wfx4rBBU97gke2nmsZqIKUaLcD23qMFeeautnCvkifSnWdmhG+wxc/d+xyAiPwRH7P4gO+pbj/9E6xQqnASd6HcphwvYSEI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860110; c=relaxed/simple; bh=ZdmP9HCAd2oDU8aGybcrgEt64oPFu3q8ZuAOXbDuT/Y=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ICxldl+lE0MBZDkhb6/HZfZgae7EN+KSoIf6msqUsBT0LxZFBAggKyO8zWy9PzbMElA1Rx9y1U7CN/eW/xCRv8ERkiZz3lT0fUFlQvrwtrj9PwJi3F+eMOdQhaIEd3yqW6NtMEJoKNvpaSngVe4Q4ZXlGX7Y4XnugdSS/BH31W4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=L+dHwV53; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="L+dHwV53" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E72AC1F00ACF; Tue, 4 Aug 2026 16:15:00 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860101; bh=4/3Q4RbdeBoEcapTLQLRtJHDgKIOcLItV3asUF3+BOc=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=L+dHwV53qP3k4wqLoSjUkrebLPJGSMNKOhTywAQQ76uECjB9szMD30Fck+oG19BQr vYFOU8xOUhiA3cVrRiA7OXhQvtzqYGMHRpMDBTFfzSMxFcdl+0RDCO6etVU6iTPQrP T/Bxvrxk5mz82p3Pfpn6J6Vszci/sWc89Ye1DtAUe6dCDnFuqk9Qh4AHDfX6sN0voA s0iu432/ah2rvpTk91k7WL9lBpZvBFecycd19POrbWL7yU8pk7pSOM6rbWl9qkjYxd +b2shnkLbbQJJJF+9mwiJPa/G/yp6a+Xaqr5P9b1zL6ASGq2XLPj8nlNynXg55D/2R CsIyERxBD5jDQ== Received: from phl-compute-02.internal (phl-compute-02.internal [10.202.2.42]) by mailfauth.phl.internal (Postfix) with ESMTP id E9FDCF40066; Tue, 4 Aug 2026 12:14:59 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-02.internal (MEProxy); Tue, 04 Aug 2026 12:14:59 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbXq 8UWtW5wdOgo9aU+rtxQ2MCFvpZ1+bx4h2YQzeYk5x0rFmmR478XZgVEan5718CRZzA7cJT RnYvxUiw3FBCFcKLF2ObIgX5tCyLTffdBvXgpfd+ITNATyzuAOQjd2N7q6QQrhazVXBuQ0 BOeytv+jFZQRffaNB2xUY8b3dffI+MBCzSsfVl7RL8OrRxgsLZ42l9oG/k+CZG9SKM1euD oLOt+3vFBxammBZr9RVXSajzGnXVlKJ4w1X2c3v/RI0Amg5PnYNtD7xpOsjyGHf3LWXSoS Oqb68hADFB8Akot3opT/e9JRdvh79gPT8zgMBkNQb93oLl3h6L9XkT20cf5g X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:14:59 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Heiko Carstens Subject: [PATCH v4 03/17] preempt: Introduce __preempt_count_{sub,add}_return() Date: Tue, 4 Aug 2026 09:14:25 -0700 Message-ID: <20260804161447.84806-4-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In order to use preempt_count() to track the interrupt disable nesting level, __preempt_count_{add,sub}_return() are introduced, as their names suggest, these primitives return the new value of the preempt_count() after changing it. The following example shows the usage of it in local_interrupt_disable(): // increase the HARDIRQ_DISABLE bit new_count =3D __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET); // if it's the first-time increment, then disable the interrupt // at hardware level. if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D HARDIRQ_DISABLE_OFFSET) { local_irq_save(flags); raw_cpu_write(local_interrupt_disable_state, flags); } Having these primitives will avoid a read of preempt_count() after changing preempt_count() on certain architectures. Acked-by: Heiko Carstens # s390 Signed-off-by: Boqun Feng --- arch/arm64/include/asm/preempt.h | 20 ++++++++++++++++++++ arch/s390/include/asm/preempt.h | 10 ++++++++++ arch/x86/include/asm/preempt.h | 10 ++++++++++ include/asm-generic/preempt.h | 14 ++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/pree= mpt.h index 932ea4b62042..9ecc2766a9f2 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -55,6 +55,26 @@ static inline void __preempt_count_sub(int val) WRITE_ONCE(current_thread_info()->preempt.count, pc); } =20 +static inline int __preempt_count_add_return(int val) +{ + u32 pc =3D READ_ONCE(current_thread_info()->preempt.count); + + pc +=3D val; + WRITE_ONCE(current_thread_info()->preempt.count, pc); + + return pc; +} + +static inline int __preempt_count_sub_return(int val) +{ + u32 pc =3D READ_ONCE(current_thread_info()->preempt.count); + + pc -=3D val; + WRITE_ONCE(current_thread_info()->preempt.count, pc); + + return pc; +} + static inline bool __preempt_count_dec_and_test(void) { struct thread_info *ti =3D current_thread_info(); diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preemp= t.h index 6e5821bb047e..0a25d4648b4c 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -139,6 +139,16 @@ static __always_inline bool should_resched(int preempt= _offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) =3D=3D preempt_of= fset); } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + return val + __atomic_add(val, &get_lowcore()->preempt_count); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return __preempt_count_add_return(-val); +} + #define init_task_preempt_count(p) do { } while (0) /* Deferred to CPU bringup time */ #define init_idle_preempt_count(p, cpu) do { } while (0) diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 578441db09f0..1220656f3370 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -85,6 +85,16 @@ static __always_inline void __preempt_count_sub(int val) raw_cpu_add_4(__preempt_count, -val); } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + return raw_cpu_add_return_4(__preempt_count, val); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return raw_cpu_add_return_4(__preempt_count, -val); +} + /* * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to resch= edule * a decrement which hits zero means we have no preempt_count and should diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index 51f8f3881523..c8683c046615 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -59,6 +59,20 @@ static __always_inline void __preempt_count_sub(int val) *preempt_count_ptr() -=3D val; } =20 +static __always_inline int __preempt_count_add_return(int val) +{ + *preempt_count_ptr() +=3D val; + + return *preempt_count_ptr(); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + *preempt_count_ptr() -=3D val; + + return *preempt_count_ptr(); +} + static __always_inline bool __preempt_count_dec_and_test(void) { /* --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 77D0346AA8F for ; Tue, 4 Aug 2026 16:15:08 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860111; cv=none; b=BAlE2S+7U39LxhLogihct+y0X5tPYSiE3iTBIwr0QRghvmjZJDoXt9kVl5815IdQgKS+Wc+cAp9XNQhGh3cq2FiaYFdJxzoCs6y/2Y+NtX8bV/h/9gh6edR8w2g/EL9WbZaZabx+YHJ0ODLrqTOSy+8nl4ODnXS03MAhlgmvzdI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860111; c=relaxed/simple; bh=X8RZfaXpnjo3eZ8Yt9kgyqp+ytrFOxLZYmWiV2ArpVs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=WfLbKNhR43Wq9ynKxkI2oxJ8YNcpY0fxutsVoqmtUe2Ikdv2jOvb2qhhssTy8ryzJLaRDjbD4WQ7jLXa9zhpK/k+IfYrUztApUakkZ0iKy4vTUIRKbBI+G2A2o+xbCFcaO5hQkNV666iKy4ommRyMycW73X0GywIwBWHVHetZjA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bpZ3hGGG; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bpZ3hGGG" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 295B01F01558; Tue, 4 Aug 2026 16:15:02 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860102; bh=Q/FRi4i2Q2qXDY7wTkxJ2+HYMjzwJRaRncVAKmZRve4=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=bpZ3hGGG0qgU1ksRPYjkk8nqSE3RdOJ/7yvC/FDuX2GJXMvBmK6Nl5GqxJj1fqp+y 6Gbcpuoj5wF7LiGD+gUS95YLudyQO25yun0I3jh2p1/NGGiIyLKdN3RHAiUPNuspmU 04V+m06v0aVCeVDIy5h7jGZRvPJmdYYvlvC8IEa0XsjD4I7mhej/2j2/FixsJDcvYi oV5Y4Ujkc2+2amM45v1prnpzfrmZ3SealhnYITu8l6ZdKYqkxB7THQOco9gvzMZch9 aH8KLgoDHXhs60HuNV6ph9EYI135uqK1+jp3iT5bOme9rTDo13icb+vaaB4ncKjJHX q8SXvcO6jfB5A== Received: from phl-compute-11.internal (phl-compute-11.internal [10.202.2.51]) by mailfauth.phl.internal (Postfix) with ESMTP id 6166CF40067; Tue, 4 Aug 2026 12:15:01 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-11.internal (MEProxy); Tue, 04 Aug 2026 12:15:01 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbNe r6KExRLiZoxOyeHN7JdW9yzjQyuaIR4UBKJYWhaJvzQko+HEf2cVybRq2l9J6a2oHQflz8 n3+y9v3TzM3d5E19jhpnSKVDeVMKItuGh1Au9gC32lk3Ey3zBiSPODgTF04T3fxxDPGikt ELBvlpTLLB1oUN3ocbitEiYgMnedu6Wanp6v/GWjdH28eVfswDY9lpUurpmPrN5lERepUT 1OXx/YhnGmVUGuaB/IdDKpMZA96GAszwR0153agU1YgOd4GFV/hO4CWd17oA4O+1L2JyUv tMAN1hCRRTwnlh27OkBpJU1Ae8R1hm4YCafasQQITpooOfrOTkpX+EpwdxZw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:00 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Stafford Horne Subject: [PATCH v4 04/17] openrisc: Include in smp.h Date: Tue, 4 Aug 2026 09:14:26 -0700 Message-ID: <20260804161447.84806-5-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul While OpenRISC currently doesn't fail to build upstream, it appears that including in the right headers is enough to break that - primarily because OpenRISC's asm/smp.h header doesn't actually provide any definition for struct cpumask. Which means the only reason we aren't failing to build the kernel is because we've been lucky enough that every spot including asm/smp.h already has definitions for struct cpumask pulled in. This became evident when trying to work on a patch series for adding ref-counted interrupt enable/disable to the kernel, where introducing a new interrupt_rc.h header suddenly introduced a build error on OpenRISC: In file included from include/linux/interrupt_rc.h:17, from include/linux/spinlock.h:60, from include/linux/mmzone.h:8, from include/linux/gfp.h:7, from include/linux/mm.h:7, from arch/openrisc/include/asm/pgalloc.h:20, from arch/openrisc/include/asm/io.h:18, from include/linux/io.h:12, from drivers/irqchip/irq-ompic.c:61: arch/openrisc/include/asm/smp.h:21:59: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration 21 | extern void arch_send_call_function_ipi_mask(const struct cpum= ask *mask); | ^~~~= ~~~ arch/openrisc/include/asm/smp.h:23:54: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration 23 | extern void set_smp_cross_call(void (*)(const struct cpumask *= , unsigned int)); | ^~~~~~~ drivers/irqchip/irq-ompic.c: In function 'ompic_of_init': >> drivers/irqchip/irq-ompic.c:191:28: error: passing argument 1 of 'set_smp_cross_call' from incompatible pointer type [-Werror=3Dincompatible-pointer-types] 191 | set_smp_cross_call(ompic_raise_softirq); | ^~~~~~~~~~~~~~~~~~~ | | | void (*)(const struct cpumask *, un= signed int) arch/openrisc/include/asm/smp.h:23:32: note: expected 'void (*)(const struct cpumask *, unsigned int)' but argument is of type 'void (*)(const struct cpumask *, unsigned int)' 23 | extern void set_smp_cross_call(void (*)(const struct cpumask *= , unsigned int)); To fix this, let's take an example from the smp.h headers of other architectures (x86, hexagon, arm64, probably more): just include linux/cpumask.h at the top. Signed-off-by: Lyude Paul Acked-by: Stafford Horne Signed-off-by: Boqun Feng --- arch/openrisc/include/asm/smp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/openrisc/include/asm/smp.h b/arch/openrisc/include/asm/sm= p.h index 007296f160ef..84653aaffa96 100644 --- a/arch/openrisc/include/asm/smp.h +++ b/arch/openrisc/include/asm/smp.h @@ -9,6 +9,8 @@ #ifndef __ASM_OPENRISC_SMP_H #define __ASM_OPENRISC_SMP_H =20 +#include + #include #include =20 --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 696244746C7 for ; Tue, 4 Aug 2026 16:15:08 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860114; cv=none; b=Bev0tiOZnsBQXgBZ+dvwWRRNCgulwZ/pTggCJ79WB2vdshDtKLkYo5jx7NP6+XCqnXtxuf140caKMlkaMUXgxVSqTuB87Q3q9kb6E1uPQspkGowzyUTx9RP6Iazi9jHINISZziRK7lmtM5I7q20jAtz1uxzR/CJZFe4KitkaZw0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860114; c=relaxed/simple; bh=d4Pss/2jrAyDtLkZJe/WHjAMnEVFYxa3hLJDJlGnfpM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=NJU2luu4xYeGYB0JzcYNn6N9gPc9GW3bANm3drmJcufkapWVXTjyk4R1WcwBuLs9RXG7M1fckrpQn1FaFun1EbeOblbDmAUTiVEvlK/tLVD84CHWNU0R8ABZU5LC3uX/eFWk9qV1iZlIz4Ij95CzUHJwU0lSpebpXaMPuEptycQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=LBGh+J4t; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="LBGh+J4t" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B19D31F00ADB; Tue, 4 Aug 2026 16:15:04 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860105; bh=4W5bJILfqvjDgPXBEUpe1QrYWZoTJ+KcU5ukL6DjGk8=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=LBGh+J4tYeDw9EDGdX4aFMcxKupU8gv9M+oPT9m57ukqQ/ckPWcc9/XJD58BaaYyV Av47M70mCJNiEcgu6S6MAuwSnmzAdfIpE3LlaZ5degoDJTC9s3QzQP0UGEjWjeaZR8 K3p6+9NGWJ6MQaNU+fFkniL4tTFF7RIFaOYheEr0hqUW4Ue25i+G01Uz84LNaxEGsU C/WUBZeSV339OBxHM3+X4+Isf91DaaP/k2ZwRVnd64JGDJtrVlS8f+K/DK1Tr7DK6Z DqVcVN9uAu4hmQfbhj5+a3EMpSQ+Y9RQfn61Jl8Ac6Dj6EEbDpQhlLQNuMo6sprL7B SA2O9OUpDeUGQ== Received: from phl-compute-03.internal (phl-compute-03.internal [10.202.2.43]) by mailfauth.phl.internal (Postfix) with ESMTP id EBFF0F40066; Tue, 4 Aug 2026 12:15:03 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-03.internal (MEProxy); Tue, 04 Aug 2026 12:15:03 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdfHSt+dGlSIgweIKZqd43ZIr2UVwMz+DLCV7Uxtqr2tWdjT0gNVbLG23y/ntMNr yg2u9xNueu4ZlCPesc7IGVMS6ZqisyNG2Hp2kDVIfbBL9QFp1F1pvbwsQcrNmamnsGJsZY AmMncETOYZEkhlG6T7JDS64AVGc7Ln4zXY6x3T0lHgb+OYvht9swwTIGsG7Xot0GJG9bWc fhVx9JgcfYh5ECBMh98FFx1XWl/VnjhK7S6I+ssCNmeXl+D6Do6MWM72zOvrRuLzi3YdIE +uwF6SMc7tQqt75yJIKNoQF+R4TbmXy2JAxyIWDxHsQYaXIqpYC7ilWb/r9+jjwAD2JcI2 jjrn0dC7npu9qXQ0szLEkd6P1uln6hxLAY/klbaun+fLCugP3BZSVJbqkiqm7hV0jyjHQ2 +Bd6oOVkP/HtUlffLLdiG6DIbuLduMANF9PYHpeNVJWR8vOmyladPWczW3XoUysFfwwOQn gBkwshgpRI3KId3zmccBCPyvq8BVw6ntQNnjjWumcky9HhOk6T7QHaxTI7BUUx7bygMYSK qhC2Kx46dV82kHGM+V7p9XCMPRxd/2J4l62vgiBdDYtkfidCirfSjEXyusSkRyk0JtAy1U Z2O3STYp3EY/wkeJ8j0Rs2UDuI2gzX3mJZWaldtYsv6JQr8WvlR1XmexiiXg X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:03 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling Date: Tue, 4 Aug 2026 09:14:27 -0700 Message-ID: <20260804161447.84806-6-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently the nested interrupt disabling and enabling is represented by _irqsave() and _irqrestore() APIs, which are relatively unsafe, for example: spin_lock_irqsave(l1, flag1); spin_lock_irqsave(l2, flag2); spin_unlock_irqrestore(l1, flags1); // accesses to interrupt-disable protected data will cause races This is even easier to trigger with guard facilities: unsigned long flag2; scoped_guard(spin_lock_irqsave, l1) { spin_lock_irqsave(l2, flag2); } // l2 locked but interrupts are enabled. spin_unlock_irqrestore(l2, flag2); (Hand-to-hand locking critical sections are not uncommon for a fine-grained lock design) And because of this unsafety, Rust cannot easily wrap the interrupt-disabling locks in a safe API, which complicates the design. To resolve this, introduce a new set of interrupt disabling APIs: * local_interrupt_disable(); * local_interrupt_enable(); They work like local_irq_save() and local_irq_restore() except that 1) the outermost local_interrupt_disable() call saves the interrupt state into a per-CPU variable, so that the outermost local_interrupt_enable() can restore the state, and 2) a per-CPU counter is added to record the nest level of these calls, so that interrupts are not accidentally enabled inside the outermost critical section. Also add the corresponding spin_lock primitives: spin_lock_irq_disable() and spin_unlock_irq_enable(), as a result, code as follows: spin_lock_irq_disable(l1); spin_lock_irq_disable(l2); spin_unlock_irq_enable(l1); // Interrupts are still disabled. spin_unlock_irq_enable(l2); doesn't have the issue that interrupts are accidentally enabled. This also makes the wrapper of interrupt-disabling locks on Rust easier to design. Signed-off-by: Lyude Paul [boqun: Apply Peter's feedback and fix spell errors reported by Ingo] Signed-off-by: Boqun Feng Suggested-by: Shrikanth Hegde --- include/linux/interrupt_rc.h | 82 ++++++++++++++++++++++++++++++++ include/linux/preempt.h | 4 ++ include/linux/spinlock.h | 23 +++++++++ include/linux/spinlock_api_smp.h | 43 +++++++++++++++++ include/linux/spinlock_api_up.h | 15 ++++++ include/linux/spinlock_rt.h | 18 +++++++ kernel/locking/spinlock.c | 31 ++++++++++++ kernel/softirq.c | 28 ++++++++++- 8 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 include/linux/interrupt_rc.h diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h new file mode 100644 index 000000000000..b9a7f05ecf42 --- /dev/null +++ b/include/linux/interrupt_rc.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_INTERRUPT_RC_H +#define __LINUX_INTERRUPT_RC_H + +/* + * include/linux/interrupt_rc.h - refcounted local processor interrupt + * management. + * + * Since the implementation of this API currently depends on + * local_irq_save()/local_irq_restore(), we split this into its own header= to + * make it easier to include without hitting circular header dependencies. + */ + +#include +#include +#include +#include + +#ifndef MODULE +/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(= ). */ +DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); + +static __always_inline void __local_interrupt_disable(void) +{ + unsigned long flags; + + local_irq_save(flags); + raw_cpu_write(local_interrupt_disable_state, flags); +} + +static __always_inline void __local_interrupt_enable(void) +{ + unsigned long flags =3D raw_cpu_read(local_interrupt_disable_state); + + local_irq_restore(flags); +} + +#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE +static __always_inline void _local_interrupt_disable(void) +{ + __local_interrupt_disable(); +} + +static __always_inline void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +#else +extern void _local_interrupt_disable(void); +extern void _local_interrupt_enable(void); +#endif + +#else /* !MODULE */ +extern void _local_interrupt_disable(void); +extern void _local_interrupt_enable(void); +#endif /* !MODULE */ + +static inline void local_interrupt_disable(void) +{ + int new_count; + + WARN_ON_ONCE(in_nmi()); + + new_count =3D hardirq_disable_enter(); + + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ + + if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D HARDIRQ_DISABLE_OFFSET) + _local_interrupt_disable(); +} + +static inline void local_interrupt_enable(void) +{ + int new_count; + + new_count =3D hardirq_disable_exit(); + + if ((new_count & HARDIRQ_DISABLE_MASK) =3D=3D 0) + _local_interrupt_enable(); +} + +#endif /* !__LINUX_INTERRUPT_RC_H */ diff --git a/include/linux/preempt.h b/include/linux/preempt.h index e2d3079d3f5f..33fc4c814a9f 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context= _level(void) #define in_softirq() (softirq_count()) #define in_interrupt() (irq_count()) =20 +#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) = >> HARDIRQ_DISABLE_SHIFT) +#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE= _OFFSET) +#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_= OFFSET) + /* * The preempt_count offset after preempt_disable(); */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 241277cd34cf..3d405cc4c121 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -57,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *= lock) __releases(lock) #endif =20 #define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock) #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock) #define raw_spin_unlock(lock) _raw_spin_unlock(lock) #define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock) +#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock) =20 #define raw_spin_unlock_irqrestore(lock, flags) \ do { \ @@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *l= ock) __releases(lock) =20 #define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lo= ck, &(flags)) =20 +#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(l= ock) + #ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ #include @@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *= lock) raw_spin_lock_irq(&lock->rlock); } =20 +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + raw_spin_lock_irq_disable(&lock->rlock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ raw_spin_lock_irqsave(spinlock_check(lock), flags); \ @@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t= *lock) raw_spin_unlock_irq(&lock->rlock); } =20 +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) __no_context_analysis +{ + raw_spin_unlock_irq_enable(&lock->rlock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsig= ned long flags) __releases(lock) __no_context_analysis { @@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spin= lock_t *lock, unsigned lon } #define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(fl= ags)) =20 +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) __no_context_analysis +{ + return raw_spin_trylock_irq_disable(&lock->rlock); +} + /** * spin_is_locked() - Check whether a spinlock is locked. * @lock: Pointer to the spinlock. diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_= smp.h index bda5e7a390cd..a5d164ac9612 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct loc= kdep_map *map) void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock); void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) __acquires(lock); +void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock); =20 unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) __acquires(lock); @@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock)= __cond_acquires(true, void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock= ); +void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releas= es(lock); void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) __releases(lock); @@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsign= ed long flags) #define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock) #endif =20 +/* Use the same config as spin_lock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ +#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE #define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock) #endif @@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsign= ed long flags) #define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock) #endif =20 +/* Use the same config as spin_unlock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ +#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(loc= k) +#endif + #ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE #define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrest= ore(lock, flags) #endif @@ -105,6 +118,18 @@ static __always_inline bool _raw_spin_trylock_irq(raw_= spinlock_t *lock) return false; } =20 +static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *= lock) + __cond_acquires(true, lock) +{ + local_interrupt_disable(); + if (_raw_spin_trylock(lock)) { + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); + return true; + } + local_interrupt_enable(); + return false; +} + static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock= , unsigned long *flags) __cond_acquires(true, lock) { @@ -143,6 +168,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t = *lock) LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); } =20 +static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + local_interrupt_disable(); + preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); + LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); +} + static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock) __no_context_analysis { @@ -188,6 +222,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_= t *lock) preempt_enable(); } =20 +static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock) + __releases(lock) +{ + spin_release(&lock->dep_map, _RET_IP_); + do_raw_spin_unlock(lock); + local_interrupt_enable(); + preempt_enable(); +} + static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock) { diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_u= p.h index a9d5c7c66e03..d03d3065ee04 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -42,6 +42,9 @@ #define __LOCK_IRQSAVE(lock, flags, ...) \ do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0) =20 +#define __LOCK_IRQ_DISABLE(lock, ...) \ + do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0) + #define ___UNLOCK_(lock) \ do { __release(lock); (void)(lock); } while (0) =20 @@ -61,6 +64,9 @@ #define __UNLOCK_IRQRESTORE(lock, flags, ...) \ do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0) =20 +#define __UNLOCK_IRQ_ENABLE(lock, ...) \ + do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0) + #define _raw_spin_lock(lock) __LOCK(lock) #define _raw_spin_lock_nested(lock, subclass) __LOCK(lock) #define _raw_read_lock(lock) __LOCK(lock, shared) @@ -70,6 +76,7 @@ #define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared) #define _raw_write_lock_bh(lock) __LOCK_BH(lock) #define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock) +#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock) #define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared) #define _raw_write_lock_irq(lock) __LOCK_IRQ(lock) #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) @@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_sp= inlock_t *lock) return 1; } =20 +static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *l= ock) + __cond_acquires(true, lock) +{ + __LOCK_IRQ_DISABLE(lock); + return 1; +} + static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock,= unsigned long *flags) __cond_acquires(true, lock) { @@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(r= wlock_t *lock, unsigned l #define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock) #define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared) #define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock) +#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock) #define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared) #define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock) #define _raw_spin_unlock_irqrestore(lock, flags) \ diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 373618a4243c..560d06384e0c 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lo= ck) rt_spin_lock(lock); } =20 +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) +{ + rt_spin_lock(lock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ @@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t= *lock) rt_spin_unlock(lock); } =20 +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) +{ + rt_spin_unlock(lock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) @@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spi= nlock_t *lock, =20 #define spin_trylock(lock) rt_spin_trylock(lock) =20 +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) +{ + return rt_spin_trylock(lock); +} + #define spin_trylock_bh(lock) rt_spin_trylock_bh(lock) =20 #define spin_trylock_irq(lock) rt_spin_trylock(lock) diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index b42d293da38b..83a17eaf5717 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##= _t *lock) \ */ BUILD_LOCK_OPS(spin, raw_spinlock, __acquires); =20 +/* No rwlock_t variants for now, so just build this function by hand */ +static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + for (;;) { + preempt_disable(); + local_interrupt_disable(); + if (likely(do_raw_spin_trylock(lock))) + break; + local_interrupt_enable(); + preempt_enable(); + + arch_spin_relax(&lock->raw_lock); + } +} + #ifndef CONFIG_PREEMPT_RT BUILD_LOCK_OPS(read, rwlock, __acquires_shared); BUILD_LOCK_OPS(write, rwlock, __acquires); @@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlo= ck_t *lock) EXPORT_SYMBOL(_raw_spin_lock_irq); #endif =20 +#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ +noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + __raw_spin_lock_irq_disable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable); +#endif + #ifndef CONFIG_INLINE_SPIN_LOCK_BH noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) { @@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spin= lock_t *lock) EXPORT_SYMBOL(_raw_spin_unlock_irq); #endif =20 +#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ +noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) +{ + __raw_spin_unlock_irq_enable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable); +#endif + #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) { diff --git a/kernel/softirq.c b/kernel/softirq.c index 10af5ed859e7..0c9b2269a8d6 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -9,6 +9,7 @@ =20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt =20 +#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE #include #include #include @@ -88,6 +89,20 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif =20 +DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); + +void _local_interrupt_disable(void) +{ + __local_interrupt_disable(); +} +EXPORT_SYMBOL(_local_interrupt_disable); + +void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +EXPORT_SYMBOL(_local_interrupt_enable); + DEFINE_PER_CPU(unsigned int, nmi_nesting); =20 /* @@ -728,10 +743,19 @@ static inline void __irq_exit_rcu(void) #endif account_hardirq_exit(current); preempt_count_sub(HARDIRQ_OFFSET); - if (!in_interrupt() && local_softirq_pending()) { + /* + * Interrupts may happen between hardirq_disable_enter() and + * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes + * softirq here, we may have a softirq handler calling + * local_interrupt_disable() but it won't disable the IRQ because + * hardirq disabling count is already 1, hence we need to prevent + * invoking softirq when a local_interrupt_disable() is ongoing. + */ + if (!in_interrupt() && !hardirq_disable_count() && + local_softirq_pending()) { /* * If we left hrtimers unarmed, make sure to arm them now, - * before enabling interrupts to run SoftIRQ. + * before enabling interrupts to run softirq. */ hrtimer_rearm_deferred(); invoke_softirq(); --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AB998474275 for ; Tue, 4 Aug 2026 16:15:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860114; cv=none; b=VOHm7sEfMnadp/VhsCtSoNBATUHEaWhjV1biRUGufGJpYpWlZHvYfG7C6wKO/oD1eG5+bnvz3kIYPyPOXnYljIqgL+P3+fsu9ZCxAqJrDfHpx7JkwfU0Y3wdbTbOsZu2UrnWVIzxVHi5jyKJwCdzXptZi+yyRqUkBJCrUyJIXm0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860114; c=relaxed/simple; bh=M6+m/0c8IuZ1xZ+UnruHtoqzFvPgWzMV2K9poAY63Gc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=rqdSIXNIMOimvlDDZ6hOMPpJ+oSJoSfxWO0LsXN1ncUY/cnoA0szZW9vPBXXTCqcvdPhyt9B/KMH4ls9bxk+13GgVtyr6Bm4VDo2PKYNf1IWyKTdQ4h8RuauYMTxU/bP3gBMa24X77s+Ju/ShQI0GNnn6VqR1Erkgm58KCSNs2g= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=gVvzibMR; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="gVvzibMR" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7CB9A1F00A3D; Tue, 4 Aug 2026 16:15:06 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860106; bh=+D9X1kYa+VRU9yRO4WbCcrFIABEuEB98nxaJDGa/Hfg=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=gVvzibMRIQCjs/BRTXDRr2eitl8/mbQqM6GGLRQhwWHKAJfvLhkEK09eeMBLII9k+ iUtJ6AApMuHdgNHC1zaXjUbMCRZrkpOPAifOozNOER/JaaNtsrJk7OLdKXSHHd8MTD 0cvQtAy8uOQSSbji/lw/sXg2qiInY9TFwRYsK/OAzKuuMjHZEAuST5qZh6hjjkWwne zvCtTCdqteJ2cvCrZNxSYz1hEV1qEe0csmBbi0MVlTfUCVwki7CEXKxs4leRySAB27 rSHJnJWGOhf83q/+o5gITd5ttVYZ9ENoNDXDfopZmcDrZ5YTmthkq0tDpebKEQYAMe SqQUExpLbFXQQ== Received: from phl-compute-10.internal (phl-compute-10.internal [10.202.2.50]) by mailfauth.phl.internal (Postfix) with ESMTP id B8B81F40066; Tue, 4 Aug 2026 12:15:05 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-10.internal (MEProxy); Tue, 04 Aug 2026 12:15:05 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbf/ yxVjFZQlSmzX0OM//kwSCAssv5Zm++y+ZiDO6tfkkrHx3qC1vS/lUGkg6f70GvmuPxRCXA 2RoP7FF6V51YYg1YoW71QMdU7eixAukmh2RL8SDYjd9aaAbgYiMsQsH9RdRHQunBp6VkNH WkS5AZM+345nNapQpKdd/jCkGx3knBBJEgYopHw+7v4iOCZ1wN95THmWRZaG8pAS2tvZbK wSgWxDkzL4SrUZrqC8nukYt+RZA/kzUQ1AR3Ao/Ybn5qXsaCYdgooyI0taknia00dMUMdN v80sNR02XnPtSChZVmQjc6/dYjix2W0r9Am/6ZY+UFkfgsIhM+Kft+c5cdhw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:04 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 06/17] irq: Add KUnit test for refcounted interrupt enable/disable Date: Tue, 4 Aug 2026 09:14:28 -0700 Message-ID: <20260804161447.84806-7-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul While making changes to the refcounted interrupt patch series, at some point on my local branch I broke something and ended up writing some kunit tests for testing refcounted interrupts as a result. So, let's include these tests now that we have refcounted interrupts. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng --- kernel/irq/Makefile | 1 + kernel/irq/refcount_interrupt_test.c | 109 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 kernel/irq/refcount_interrupt_test.c diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 86a2e5ae08f9..44c4d6fc502a 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) +=3D affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) +=3D debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) +=3D matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) +=3D irq_test.o +obj-$(CONFIG_KUNIT) +=3D refcount_interrupt_test.o diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_int= errupt_test.c new file mode 100644 index 000000000000..ca904dba24b9 --- /dev/null +++ b/kernel/irq/refcount_interrupt_test.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for refcounted interrupt enable/disables. + */ + +#include +#include + +#define TEST_IRQ_ON() KUNIT_EXPECT_FALSE(test, irqs_disabled()) +#define TEST_IRQ_OFF() KUNIT_EXPECT_TRUE(test, irqs_disabled()) + +/* =3D=3D=3D=3D=3D Test cases =3D=3D=3D=3D=3D */ +static void test_single_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); +} + +static void test_nested_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_multiple_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_irq_save(struct kunit *test) +{ + unsigned long flags; + + local_irq_save(flags); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_irq_save(flags); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static struct kunit_case test_cases[] =3D { + KUNIT_CASE(test_single_irq_change), + KUNIT_CASE(test_nested_irq_change), + KUNIT_CASE(test_multiple_irq_change), + KUNIT_CASE(test_irq_save), + {}, +}; + +/* init and exit are the same. */ +static int test_init(struct kunit *test) +{ + TEST_IRQ_ON(); + + return 0; +} + +static void test_exit(struct kunit *test) +{ + TEST_IRQ_ON(); +} + +static struct kunit_suite refcount_interrupt_test_suite =3D { + .name =3D "refcount_interrupt", + .test_cases =3D test_cases, + .init =3D test_init, + .exit =3D test_exit, +}; + +kunit_test_suite(refcount_interrupt_test_suite); +MODULE_AUTHOR("Lyude Paul "); +MODULE_DESCRIPTION("Refcounted interrupt unit test suite"); +MODULE_LICENSE("GPL"); --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9B3F947427C for ; Tue, 4 Aug 2026 16:15:11 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860115; cv=none; b=FNhpKTp3WYRSZ9Dl+Aat/FbcVDzOJ1M4NIwnn1tPl5yZiFj21i0sjjh1pPCBYlw/kpVFSY93ruHlSpZMy3RZ94EAsI5k+94jFNVexaRGJROZlw+tYMH5FDhV0ADDUxIgORzZxop72+SotVjnyIKITyREJZew6Fw/1r1yuJBP9vM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860115; c=relaxed/simple; bh=ez8mh8KYXsxEZezHI84ye4skuWKbHPcxR5bfxvHXALU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=p9q7st2lk0O4xDc/zpFWnxBDC0ajBQUNS/7Eke4tTyvk6N2Av5sieWLqCJJxaNmVGp1z7xd519QeZAX8+cJJROOh4RlTdiyQx1y5FhgaCVWpgk835byKhazbKuyr1LLAvl2fdDuBkGhrIu1VDzCEw4+YEacY1iZDr3ZVCr8lcus= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=YQp911EF; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="YQp911EF" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 2EED41F000E9; Tue, 4 Aug 2026 16:15:08 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860108; bh=fJJjzop4LECjkHzK46lASg2zu9hWEBiVBs7rN6nvoKc=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=YQp911EFRdny3wLlSrq/lcVTkWfxKkERHi9PPssFucUSMBm2Rl30dck2AlV9hsCh7 +SDszV43ygIum5IKbw47FPOmWc3kMad8T7CVKpxfOcQhqzmDAUo78J3BaEKt29neKg OKw/9QkgZYKICmsgRI20iD7mUF7m3CdeVgRLRHsw2exZyGMPr5WjpxbJVeA53JbDiu J1GuIhfQDjCo1uz+4fA4AbMkxhBiNWpyHUTPCM4t4e32objF1rT8UUWfiBj5ldTJcu Z/0j7inLJ/0HLjHDw9WDDCcKrW19qHzftRXFV4Wtm3x6w1KnPDviJUe9Yj2OY0trOy ptrAgY9MoJilw== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id 6D477F40066; Tue, 4 Aug 2026 12:15:07 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-06.internal (MEProxy); Tue, 04 Aug 2026 12:15:07 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbTv /tZiNGE041QSoQoleXBD2zzVVS8COIP//K06XOEDHhb1OjBYtERM67VN/fXGRN2GYjWBuR E+wgljilAj57wTBUc5ZC0YqFMaEc1Eg5KM3WySfyC5wytp6Ep1nMqJrklrMG3kf6c4lFDX ugsB0lRdJEBK7yuccrVuR3B8H5+PxFjusOMtGwizslZfTKlnN7t4U2RQtr5qKzKoywNorx B8pFXbiTtMsRJk9czxv6rue3a+1ah/cfVVrqRPVHR3C9ALz7rXK6Jmtxhr8PigzNp+KoCQ bJAnH4rYIvbvUrYxZ3xAJOVX8Jt+gxea3ZeO/GMTOgEGU/HNKXdhovj8ouEw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:06 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 07/17] locking: Switch to _irq_{disable,enable}() variants in cleanup guards Date: Tue, 4 Aug 2026 09:14:29 -0700 Message-ID: <20260804161447.84806-8-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The semantics of various IRQ disabling guards match what *_irq_{disable,enable}() provide, i.e. the interrupt disabling is properly nested, therefore it's OK to switch to use *_irq_{disable,enable}() primitives. [boqun: Adjust the user-side changes in do_sched_cfs_*_timer() provided by Peter and Lyude] Signed-off-by: Boqun Feng --- include/linux/spinlock.h | 26 ++++++++++++-------------- kernel/sched/fair.c | 12 ++++++------ 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 3d405cc4c121..799a8f7d2741 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -572,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acq= uires(_T), __releases(*(raw #define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(= raw_spinlock_nested, _T) =20 DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t, - raw_spin_lock_irq(_T->lock), - raw_spin_unlock_irq(_T->lock)) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(= raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw= _spinlock_irq, _T) =20 -DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->= lock)) +DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disa= ble(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __release= s(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS= (raw_spinlock_irq_try, _T) =20 @@ -592,14 +592,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acq= uires(_T), __releases(*(raw #define class_raw_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(= raw_spinlock_bh_try, _T) =20 DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t, - raw_spin_lock_irqsave(_T->lock, _T->flags), - raw_spin_unlock_irqrestore(_T->lock, _T->flags), - unsigned long flags) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __release= s(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS= (raw_spinlock_irqsave, _T) =20 DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try, - raw_spin_trylock_irqsave(_T->lock, _T->flags)) + raw_spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __rel= eases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_A= TTRS(raw_spinlock_irqsave_try, _T) =20 @@ -618,13 +617,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_= T), __releases(*(spinlock_t #define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinloc= k_try, _T) =20 DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t, - spin_lock_irq(_T->lock), - spin_unlock_irq(_T->lock)) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spin= lock_t **)_T)) #define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinloc= k_irq, _T) =20 DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try, - spin_trylock_irq(_T->lock)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(= spinlock_t **)_T)) #define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spi= nlock_irq_try, _T) =20 @@ -640,14 +639,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquire= s(_T), __releases(*(spinloc #define class_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spin= lock_bh_try, _T) =20 DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t, - spin_lock_irqsave(_T->lock, _T->flags), - spin_unlock_irqrestore(_T->lock, _T->flags), - unsigned long flags) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(= spinlock_t **)_T)) #define class_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spi= nlock_irqsave, _T) =20 DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try, - spin_trylock_irqsave(_T->lock, _T->flags)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __release= s(*(spinlock_t **)_T)) #define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS= (spinlock_irqsave_try, _T) =20 diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d78467ec6ee1..a46c4ff49f74 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7120,7 +7120,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwid= th *cfs_b) * period the timer is deactivated until scheduling resumes; cfs_b->idle is * used to track this state. */ -static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int over= run, unsigned long flags) +static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int over= run) __must_hold(&cfs_b->lock) { int throttled; @@ -7155,10 +7155,10 @@ static int do_sched_cfs_period_timer(struct cfs_ban= dwidth *cfs_b, int overrun, u * This check is repeated as we release cfs_b->lock while we unthrottle. */ while (throttled && cfs_b->runtime > 0) { - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); + raw_spin_unlock_irq_enable(&cfs_b->lock); /* we can't nest cfs_b->lock while distributing bandwidth */ throttled =3D distribute_cfs_runtime(cfs_b); - raw_spin_lock_irqsave(&cfs_b->lock, flags); + raw_spin_lock_irq_disable(&cfs_b->lock); } =20 /* @@ -7266,7 +7266,7 @@ static __always_inline void return_cfs_rq_runtime(str= uct cfs_rq *cfs_rq) static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) { /* confirm we're still not at a refresh boundary */ - scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) { + scoped_guard(raw_spinlock_irq, &cfs_b->lock) { u64 runtime =3D 0, slice =3D sched_cfs_bandwidth_slice(); =20 cfs_b->slack_started =3D false; @@ -7351,14 +7351,14 @@ static enum hrtimer_restart sched_cfs_period_timer(= struct hrtimer *timer) int idle =3D 0; int count =3D 0; =20 - CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock); + guard(raw_spinlock_irq)(&cfs_b->lock); =20 for (;;) { overrun =3D hrtimer_forward_now(timer, cfs_b->period); if (!overrun) break; =20 - idle =3D do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags); + idle =3D do_sched_cfs_period_timer(cfs_b, overrun); =20 if (++count > 3) { u64 new, old =3D ktime_to_ns(cfs_b->period); --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E9FF847255F; Tue, 4 Aug 2026 16:15:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860115; cv=none; b=ix7AoqD1EoquotMiw24g6QgUWsOKa1WwPMG2+el2ePlDl57OQ9yesubn4+Ht5m0CQgeka/JJq+ykhBV2AwwERL3OzEOvm2VDm6gzLktqGYXD6IhKEK8zpQVPytz46vXRKCuNZTy2uavKV5S0Cv6xZ9PHBsV7AUBkAhA9W8ZmHOU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860115; c=relaxed/simple; bh=ec5Ah/8+IE+sh6duXoIfePrYPk/nTB/nNJ9FI66HfV4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=IZyaEKY9mqAiW+DAhpC024s2dRqMd90AHyWd0j9Q+r1qsHTUuHfI9KwZ/A3mfFLXWFypemGnKwVJbTFUWOv5bKVJ/KgYE+kWXaBHAM6m9NdEvG/F2qaZlX+hE/6bQu9mrqNVZrg7vWZvL4Rb9Nz0zLTwyX1bdAygfJ8Dzywl2kU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Pbb3RLXk; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Pbb3RLXk" Received: by smtp.kernel.org (Postfix) with ESMTPSA id AE6301F00AC4; Tue, 4 Aug 2026 16:15:09 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860110; bh=0thUqhS5l/CCHvciQUIdxDtNj6hdUCGVYjpithCalyQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Pbb3RLXkyr2Y24GmRUp92u9x138YwQgyd+e8glIkFYfBKmoVO8FNwnNP/eeiDwIxg iiqI0NMQJfJ61PFmzVKB5duugweh5hT0KecAKR7ekaeSIc/LwZxYOHUzRiNAZVjDLw rxtECgHunow9RPnfKS5qhiKPY6t0gxpTeylOPYyMQcE7Fuwlpc9xKTrQsXzyoX79tn 90KElkLMKhY9GA1XqPAaqhs6lvVIgvASIUd0NJkPIaIgCMPAYwQNMKafJiVddF+WDo HnJCf+mjkoR1x6YKAScQy2eA//uAeuNivSjdlvQn1o1aJEOg9MaV9MX5m+fMuVpjVv zT8ke91u9Jxgg== Received: from phl-compute-06.internal (phl-compute-06.internal [10.202.2.46]) by mailfauth.phl.internal (Postfix) with ESMTP id EC10EF40067; Tue, 4 Aug 2026 12:15:08 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-06.internal (MEProxy); Tue, 04 Aug 2026 12:15:08 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbrS T4n7Yy/xnbst++Q/7UwywmdIB2YHsJedvBGrhDBbxijp+KAZ7iBacxQdpd36fElvRz57Hs 9dUk8MVn6xcIdkdfjBL0rTuotTXsHq23D12GbBZL2a0tK0esWgSxaaeOJHxhLmpBg8cT/y EPvov8JtezK4Nw/HRE6vy+0HqxCJLezdF39bf80hR00agqKJv6awtKDOuxR5ZcCDAVOgiC DM78SHG75Kg/k0yq9V7zqIhTvBxqvOSexKC0/g9V+mqVhrkXQu1h6eiEGK5XSdye3Nr1gb 8OXd3GTIqqSJ1vjLdYN/zVS0pGlxJ59CKgctv22uJI+DG6Ny0qY0sP8c6uzA X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:08 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 08/17] sched: Remove the unused preempt_offset parameter of __cant_sleep() Date: Tue, 4 Aug 2026 09:14:30 -0700 Message-ID: <20260804161447.84806-9-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The preempt_offset is always 0 in all the callsites of __cant_sleep(), hence remove it. It also allows us to clear up the code a bit by no longer using a "preempt_count() > .." comparison. Signed-off-by: Boqun Feng --- include/linux/kernel.h | 4 ++-- kernel/sched/core.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index e5570a16cbb1..24414c79e59a 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -72,7 +72,7 @@ extern int dynamic_might_resched(void); #ifdef CONFIG_DEBUG_ATOMIC_SLEEP extern void __might_resched(const char *file, int line, unsigned int offse= ts); extern void __might_sleep(const char *file, int line); -extern void __cant_sleep(const char *file, int line, int preempt_offset); +extern void __cant_sleep(const char *file, int line); extern void __cant_migrate(const char *file, int line); =20 /** @@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line); * this macro will print a stack trace if it is executed with preemption e= nabled */ # define cant_sleep() \ - do { __cant_sleep(__FILE__, __LINE__, 0); } while (0) + do { __cant_sleep(__FILE__, __LINE__); } while (0) # define sched_annotate_sleep() (current->task_state_change =3D 0) =20 /** diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..aa116daf21bd 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9199,7 +9199,7 @@ void __might_resched(const char *file, int line, unsi= gned int offsets) } EXPORT_SYMBOL(__might_resched); =20 -void __cant_sleep(const char *file, int line, int preempt_offset) +void __cant_sleep(const char *file, int line) { static unsigned long prev_jiffy; =20 @@ -9209,7 +9209,7 @@ void __cant_sleep(const char *file, int line, int pre= empt_offset) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; =20 - if (preempt_count() > preempt_offset) + if (preempt_count()) return; =20 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 96C504746B8 for ; Tue, 4 Aug 2026 16:15:12 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860116; cv=none; b=PFZTn+Ln9wUlU1ex3fLhIVoYjvkS/ym+Ht4L0PBedFlHdYI/qDOPj7gGBnTttFrYiCxKQtIYbtpmsb1LAaDIgKOVg0vNclV2T9jpsybsHBTHOH40EbRD+rfqrXHl/O7Q/e/gpfPtKntQgCt7YJC3TY48PJ6eSwnCEDv8v6UDmy4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860116; c=relaxed/simple; bh=gdX2/8yO0N7n17FlM8ctDDZ1lMRo6f/CFk/F/ffOAD4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=DR3XBrYjUMc6Dxi0OSnrK97Z3gNw3SnwrvKKE4MkMgsZiO9/3vYHc4n/HAtJf9ASlzrPmyIjIHmJfdnhoPkYQkgrlrqfTMc+3UfxqmZ67rWkLHfyXH2DYwbqVJS//ZPEO/XeTDq+SRx2N2n2inNP+vjwo4T4fkhPlcbEahbblRM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Vbo5Fc3x; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Vbo5Fc3x" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3BB161F00ADF; Tue, 4 Aug 2026 16:15:11 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860111; bh=uaMiBt5vafGBdINwxuR8k+kc9W7/u2yo8z9BluwAE/g=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Vbo5Fc3xiyvVNnXaKZYLgOJiD573SdOdZCCuIzEH9LpoMIwFdiKCxYGEzj6zk2uah DMoJV8OStdVhObep1NyOcQfcAtDyUm8pleq9M9Pyrijy4AwlXZwfiCv2TTBptlI/YH FdA821vuw9v9hs2qDZfC2HS/Do8t6+u5BE1frMOduSov3DvAHkejg6MvY6TmXVtvmq FL92vzxQhWN/7vWa4lHB3LggTqWeJ6X5ydTyr+QMHOIBmXSw9INCLgqkD4cSoYjacq Eu9EMEJmbcnDtWGisRYgxiunE3Na6BG/HnFmN4sD/pGCeKVEN/zoXQGpjRzS41/vyy WjYAj6qyjo/sA== Received: from phl-compute-02.internal (phl-compute-02.internal [10.202.2.42]) by mailfauth.phl.internal (Postfix) with ESMTP id 78A21F40066; Tue, 4 Aug 2026 12:15:10 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-02.internal (MEProxy); Tue, 04 Aug 2026 12:15:10 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdfHSt+dGlSIgweIKZqd43ZIr2UVwMz+DLCV7Uxtqr2tWdjT0gNVbLG23y/ntMNr yg2u9xNueu4ZlCPesc7IGVMS6ZqisyNG2Hp2kDVIfbBL9QFp1F1pvbwsQcrNmamnsGJsZY AmMncETOYZEkhlG6T7JDS64AVGc7Ln4zXY6x3T0lHgb+OYvht9swwTIGsG7Xot0GJG9bWc fhVx9JgcfYh5ECBMh98FFx1XWl/VnjhK7S6I+ssCNmeXl+D6Do6MWM72zOvrRuLzi3YdIE +uwF6SMc7tQqt75yJIKNoQF+R4TbmXy2JAxyIWDxHsQYaXIqpYC7ilWb/r9+jjwAD2JcNX ZHW6YG9QXu50mMhrSTuVu5sHfh2gb0c7rW3JeGWGaD/0HvmORg07SFbFbH3Ybq6WCUTkJv LPQlEkSTsZlPCfG8f45vVU0jlIX/gCp5dSSXCj4KlsKgeEdVj7uHtnsXYoAhhrAOyNL4W5 ofxD+1xC6EXrxGrggTEvjUX7WwhHRdQp8w9lnfdBuSUUC5uExYHswUHnY+1r05+0BXad2v fwz23DqRQpz9LCzZ8dndZPpgWnOyjaMbcRH+O88Q6ZMgV+Ws7MiH02fUoXK4V9UITeX0Dd bqWIfGhEg7xuDLV1rff26bPb9TAoiD/tVo6qdsAm15Db8f8qfl1MXBwIwKMA X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:09 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 09/17] sched: Avoid signed comparison of preempt_count() in __cant_migrate() Date: Tue, 4 Aug 2026 09:14:31 -0700 Message-ID: <20260804161447.84806-10-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently preempt_count() is always a non-negative int on all archs (PREEMPT_NEED_RESCHED archs will mask out the MSB when returning preempt_count()), hence the checking in __cant_migrate() is in fact just checking whether preempt_count() is 0 or not. In a future change, we are going to use all the 32 bits of preempt_count(), which would make negative int values possible from preempt_count(). Therefore convert the "> 0" comparison into a zero check to prepare for the future change. No functional changes are intended. Signed-off-by: Boqun Feng --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index aa116daf21bd..9b3f1764fa9e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9241,7 +9241,7 @@ void __cant_migrate(const char *file, int line) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; =20 - if (preempt_count() > 0) + if (preempt_count()) return; =20 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A84504756D8 for ; Tue, 4 Aug 2026 16:15:15 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860122; cv=none; b=ck5mCbZXGifdP5nWVyvyDBd3pB4+cM8PPGrJNz1XBzWZa38p4diOTJJUANh1HIL6AF36nqDGmzH8TmyZPeH80KipUTWHblZbMFmA5V+hDnrv03HLs/KdzOFGmvaJ8wQaJUL2kYtmi4CuY88XLj6cjCacxx06rWSr+bQZ80xKDO0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860122; c=relaxed/simple; bh=qK0fYEr3n4z2PgwKv85T/U0P8sMWyDiOz6yn2hRuZkA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=hPCn1ICq270Ae971cQvjtEw8iV7JLYfT7k/GzFspeLVeXtBkbGT+wj456VrIIX4+dfSK7LeTMUHsQza8O6Ow3ro4/AE1QffNVx8F8w6QLfpkAVb7zR3/i7TB+6cpUIh0VNfQzsDsiLez+QUDnX5PLwv2LQjJkt8OFS7alZgzYWQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=IWJ7+osN; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="IWJ7+osN" Received: by smtp.kernel.org (Postfix) with ESMTPSA id CCE9E1F01558; Tue, 4 Aug 2026 16:15:12 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860113; bh=6tPZdbvN6QnDZI1t01nqX2384FZjLhK5No57sQfgowU=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=IWJ7+osN+LS3iCHd5gE1j0TiluYum1o7/ATtv5W8gGr57kT6GVj5jjc11lBxvu7oW 8wcNyARDW3Jz8hzumTiZJT44UTdVZdMF+B6WO9vPMb6KSUDmxBXF+pxfoLmr8RUkBL N2j/1b6CiR10L+eYTky9aQNLsvXHwLje7tb9fS4G1v3oZf6C4JpuJgJDfDpD4Qq7yE mqle+zdBZDvj7QQsN2zORltph4F6yCZyuvR6q5f8469sfZd1kxZGQv5ytRKa1SWZDm K2R06m3xBY0CVhdZ5NM0zVMWxXNllmQ4x8KJequc6faFK4Dx7ErMTP9yFutioJOHnc EDVGOYahggu4A== Received: from phl-compute-03.internal (phl-compute-03.internal [10.202.2.43]) by mailfauth.phl.internal (Postfix) with ESMTP id 1719FF40066; Tue, 4 Aug 2026 12:15:12 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-03.internal (MEProxy); Tue, 04 Aug 2026 12:15:12 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTE6f1b/PLD7WfUEjv3UPH0IuooDzdONFOoQTajOMPpDZsgOHZQOVqJ4v+RHATO1sj EZSfNibFX853su8vPqAqnUSuvHiUbplQY0egsm0mTGT1lvu1S/5Nh/tuqaumzTAbEEqn88 A4HuRrhsSYsTxma98RYyzM2Gbo4E2wg96qbDSuNUia/kyZtd9PuUf9ikEEX+FWBaYhk2LZ 4A6T9U94iL3j6tNfmRpUMp/Z8p6+0GWHZbXf2pAo98Kg1ESY7btybSSSUnrcHypkA1CoVm T+WboQPfBLNhjm5E0DtWi31bTX/atiaNKg8ICM8aktklu407EF+Nx0w3ySFDe9rX99/5kE 0eTYdYpwi1Bjvc96Ixfm7dnLWCAXmnf4Azheezsj4bDL0Exi9u0eu7TFEUzyVWsiEmq/9e f8+g7F8F8ms6gIXR20Tb7aVtG9owNbWsbUEj+EHUoZVxB4z7lH4l6xr1LaENU/JkmQ6rVu ongGFtSs49LZNZh06uq3JkUjqlW6X0ey9GbZ7VmjRLfp+v7zrbGpMJ4wMHQvFNFN9wh7H1 vSc3Q6/Ig3YKCNuNgjyOT0kZiTiVawZc4/RkIkHJOgIbflWNbkVdx+ovEE65+YhwqU4P0j U9uZbDu5ZvCQ+s2KGXkXe05wK2utVv0cZHhsGzvRSdUmF0QmgdPeMIW/jiBQ X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:11 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Tue, 4 Aug 2026 09:14:32 -0700 Message-ID: <20260804161447.84806-11-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" With the changes that enable preempt count to track IRQ disabling nesting, we don't have enough bits in 32-bit preempt count implementation, as a result we move NMI nesting bits out of the 32-bit preempt count. However on the architectures that can support 64-bit preempt count implementation, we can keep the NMI nesting bits in the 32-bit preempt count and avoid maintaining NMI nesting bits outside of the same cache line. Therefore HAS_SEPARATE_PREEMPT_RESCHED_BITS is introduced to allow architectures to select this. Note that under this Kconfig, preempt count is maintained in a 64-bit word however preempt_count() still remains as an int because all the effective bits still fit in (previously we mask out NEED_RESCHED bit in preempt_count()). This should make no functional changes for existing preempt_count() users. Enable this for x86_64 along with the introduction of the Kconfig. [boqun: Undo the __preempt_count_{add,sub}() optimization in 32-bit preempt count since it may introduce {over,under}flow] Originally-by: Peter Zijlstra Signed-off-by: Boqun Feng --- arch/x86/Kconfig | 1 + arch/x86/include/asm/preempt.h | 55 +++++++++++++++++++++++----------- arch/x86/kernel/cpu/common.c | 2 +- include/linux/hardirq.h | 47 +++++++++++++++++++++-------- include/linux/preempt.h | 23 ++++++++++++-- kernel/Kconfig.preempt | 4 +++ kernel/sched/core.c | 12 ++++++-- kernel/softirq.c | 6 ++++ lib/locking-selftest.c | 2 +- 9 files changed, 115 insertions(+), 37 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..6a7067d20a6a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -326,6 +326,7 @@ config X86 select USER_STACKTRACE_SUPPORT select HAVE_ARCH_KCSAN if X86_64 select PROC_PID_ARCH_STATUS if PROC_FS + select HAS_SEPARATE_PREEMPT_RESCHED_BITS if X86_64 && PREEMPT_COUNT select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX select FUNCTION_ALIGNMENT_16B if X86_64 || X86_ALIGNMENT_16 select FUNCTION_ALIGNMENT_4B diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 1220656f3370..12353eeebc52 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -7,10 +7,20 @@ =20 #include =20 -DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count); +DECLARE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count); =20 -/* We use the MSB mostly because its available */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* + * We use the MSB for PREEMPT_NEED_RESCHED mostly because it is available. + */ +#define PREEMPT_NEED_RESCHED (~(((unsigned long)-1L) >> 1)) + +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +#define __pc_dec "decq" +#define __pc_op(op, ...) raw_cpu_##op##_8(__VA_ARGS__) +#else +#define __pc_dec "decl" +#define __pc_op(op, ...) raw_cpu_##op##_4(__VA_ARGS__) +#endif =20 /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -24,18 +34,26 @@ DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count); */ static __always_inline int preempt_count(void) { - return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED; + return __pc_op(read, __preempt_count) & ~PREEMPT_NEED_RESCHED; } =20 -static __always_inline void preempt_count_set(int pc) +/* + * unsigned long preempt count parameter works for both 32bit and 64bit ca= ses: + * + * - For 32bit, "int" (the return of preempt_count()) and "unsigned long" = have + * the same size. + * - For 64bit, the effective bits of a preempt count sits in 32bit, and we + * reserve the NEED_RESCHED bit from the old count. + */ +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; =20 - old =3D raw_cpu_read_4(__preempt_count); + old =3D __pc_op(read, __preempt_count); do { new =3D (old & PREEMPT_NEED_RESCHED) | (pc & ~PREEMPT_NEED_RESCHED); - } while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new)); + } while (!__pc_op(try_cmpxchg, __preempt_count, &old, new)); } =20 /* @@ -58,17 +76,17 @@ static __always_inline void preempt_count_set(int pc) =20 static __always_inline void set_preempt_need_resched(void) { - raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED); + __pc_op(and, __preempt_count, ~PREEMPT_NEED_RESCHED); } =20 static __always_inline void clear_preempt_need_resched(void) { - raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED); + __pc_op(or, __preempt_count, PREEMPT_NEED_RESCHED); } =20 static __always_inline bool test_preempt_need_resched(void) { - return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED); + return !(__pc_op(read, __preempt_count) & PREEMPT_NEED_RESCHED); } =20 /* @@ -77,22 +95,22 @@ static __always_inline bool test_preempt_need_resched(v= oid) =20 static __always_inline void __preempt_count_add(int val) { - raw_cpu_add_4(__preempt_count, val); + __pc_op(add, __preempt_count, val); } =20 static __always_inline void __preempt_count_sub(int val) { - raw_cpu_add_4(__preempt_count, -val); + __pc_op(add, __preempt_count, -val); } =20 static __always_inline int __preempt_count_add_return(int val) { - return raw_cpu_add_return_4(__preempt_count, val); + return __pc_op(add_return, __preempt_count, val); } =20 static __always_inline int __preempt_count_sub_return(int val) { - return raw_cpu_add_return_4(__preempt_count, -val); + return __pc_op(add_return, __preempt_count, -val); } =20 /* @@ -102,7 +120,7 @@ static __always_inline int __preempt_count_sub_return(i= nt val) */ static __always_inline bool __preempt_count_dec_and_test(void) { - return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e, + return GEN_UNARY_RMWcc(__pc_dec, __my_cpu_var(__preempt_count), e, __percpu_arg([var])); } =20 @@ -111,7 +129,7 @@ static __always_inline bool __preempt_count_dec_and_tes= t(void) */ static __always_inline bool should_resched(int preempt_offset) { - return unlikely(raw_cpu_read_4(__preempt_count) =3D=3D preempt_offset); + return unlikely(__pc_op(read, __preempt_count) =3D=3D preempt_offset); } =20 #ifdef CONFIG_PREEMPTION @@ -158,4 +176,7 @@ do { \ =20 #endif /* PREEMPTION */ =20 +#undef __pc_op +#undef __pc_dec + #endif /* __ASM_PREEMPT_H */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3df21d26460..73a6d9f6a78e 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2236,7 +2236,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, curren= t_task) =3D &init_task; EXPORT_PER_CPU_SYMBOL(current_task); EXPORT_PER_CPU_SYMBOL(const_current_task); =20 -DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) =3D INIT_PREEMPT_COUNT; +DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) =3D INIT_PREEMPT_= COUNT; EXPORT_PER_CPU_SYMBOL(__preempt_count); =20 DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) =3D TOP_= OF_INIT_STACK; diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 8d4895531a45..73b48dd9f135 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -10,8 +10,6 @@ #include #include =20 -DECLARE_PER_CPU(unsigned int, nmi_nesting); - extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); =20 @@ -94,6 +92,37 @@ void irq_exit_rcu(void); #define arch_nmi_exit() do { } while (0) #endif =20 +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +static __always_inline void __preempt_count_nmi_enter(void) +{ + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); +} + +static __always_inline void __preempt_count_nmi_exit(void) +{ + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); +} +#else +DECLARE_PER_CPU(unsigned int, nmi_nesting); + +#define __preempt_count_nmi_enter() \ + do { \ + __preempt_count_add(HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ + __this_cpu_inc(nmi_nesting); \ + preempt_count_set(preempt_count() | NMI_MASK); \ + } while (0) + +#define __preempt_count_nmi_exit() \ + do { \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + if (!__this_cpu_dec_return(nmi_nesting)) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ + } while (0) + +#endif + /* * NMI vs Tracing * -------------- @@ -110,18 +139,14 @@ void irq_exit_rcu(void); do { \ lockdep_off(); \ arch_nmi_enter(); \ - /* Maximum NMI nesting is 15. */ \ - BUG_ON(__this_cpu_read(nmi_nesting) >=3D 15); \ - __this_cpu_inc(nmi_nesting); \ - __preempt_count_add(HARDIRQ_OFFSET); \ - preempt_count_set(preempt_count() | NMI_MASK); \ + __preempt_count_nmi_enter(); \ } while (0) =20 #define nmi_enter() \ do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ - ct_nmi_enter(); \ + ct_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ @@ -129,12 +154,8 @@ void irq_exit_rcu(void); =20 #define __nmi_exit() \ do { \ - unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(HARDIRQ_OFFSET); \ - nesting =3D __this_cpu_dec_return(nmi_nesting); \ - if (!nesting) \ - preempt_count_set(preempt_count() & ~NMI_MASK); \ + __preempt_count_nmi_exit(); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 33fc4c814a9f..8299657f0f86 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -34,14 +34,31 @@ * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_DISABLE_MASK: 0x00ff0000 * HARDIRQ_MASK: 0x0f000000 + * + * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=3Dy, PREEMPT_NEED_RESCHED is put= in a + * separate word and that allows 64bit load-store architectures to 'set' + * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric + * modifications used on preempt_count and still load the whole thing + * (single-copy) atomically, without having to resort to full atomic + * operations. + * + * Because of the above, NMI_MASK bits are different depending on + * HAS_SEPARATE_PREEMPT_RESCHED_BITS: + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=3Dn: + * * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=3Dy: + * NMI_MASK: 0xf0000000 + * (PREEMPT_NEED_RESCHED is in a different word) */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 1 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BIT= S)) =20 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) @@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_= level(void) * preempt_count() is commonly implemented with READ_ONCE(). */ =20 -#define nmi_count() (preempt_count() & NMI_MASK) -#define hardirq_count() (preempt_count() & HARDIRQ_MASK) +#define nmi_count() (preempt_count() & NMI_MASK) +#define hardirq_count() (preempt_count() & HARDIRQ_MASK) #ifdef CONFIG_PREEMPT_RT # define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK) # define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | sof= tirq_count()) diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 88c594c6d7fc..35f546a042b1 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK config PREEMPT_COUNT bool =20 +config HAS_SEPARATE_PREEMPT_RESCHED_BITS + bool + depends on PREEMPT_COUNT && 64BIT + config PREEMPTION bool select PREEMPT_COUNT diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9b3f1764fa9e..6d88343c3bad 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5973,8 +5973,13 @@ void preempt_count_add(int val) #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? + * + * Cannot detect underflow based on the current preempt_count() value + * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes= all 32 + * bits. */ - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) && + DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) return; #endif __preempt_count_add(val); @@ -6006,7 +6011,10 @@ void preempt_count_sub(int val) /* * Underflow? */ - if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) + unsigned int uval =3D val; + unsigned int pc =3D preempt_count(); + + if (DEBUG_LOCKS_WARN_ON(pc - uval > pc)) return; /* * Is the spinlock portion underflowing? diff --git a/kernel/softirq.c b/kernel/softirq.c index 0c9b2269a8d6..7980a4a232f9 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -103,7 +103,13 @@ void _local_interrupt_enable(void) } EXPORT_SYMBOL(_local_interrupt_enable); =20 +#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +/* + * Any 32bit architecture that still cares about performance should + * probably ensure this is near preempt_count. + */ DEFINE_PER_CPU(unsigned int, nmi_nesting); +#endif =20 /* * SOFTIRQ_OFFSET usage: diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index bfafe1204c7b..c3d976c801bb 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures; =20 static void dotest(void (*testcase_fn)(void), int expected, int lockclass_= mask) { - int saved_preempt_count =3D preempt_count(); + long saved_preempt_count =3D preempt_count(); #ifdef CONFIG_PREEMPT_RT int saved_mgd_count =3D current->migration_disabled; int saved_rcu_count =3D current->rcu_read_lock_nesting; --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 542F54756B4 for ; Tue, 4 Aug 2026 16:15:15 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860118; cv=none; b=Sw20YwOZVQoZxoyACFtcaiicBl88kMFWS7ERhdXcxFCFklPbisumPlX1dFW6OCJ56PdG0rEWcPaaOKX3ncO2pmDvao0ng28p4NoJZZYzO4zuaCCSHbbHpR/cnKIqlH9NO2fSpL8/o5dj5e/yX3WvUMcRfEEL3db7fdACTlUm+rU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860118; c=relaxed/simple; bh=JqcH/mibcR84zB0h7wBnVEuD0X2zbft/81pdfe10n0s=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=j/jB4Ql+0DzUllOk6dXS75TPeeFy4Jh1zWdVYd8VPtuiJnGtmqaeCpRHMm8KquOdViNoy44mEg9v5o1qdzSY4sHec7WilAl5Lp9ZdkKMm4FhroxAQnTWBcyRY02DDk6LzT3NmI6BpfGFiHglXVJTzR6coGoAI8DM7Iw5gf5XP3A= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=hkF7ch6E; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="hkF7ch6E" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6486F1F00A3D; Tue, 4 Aug 2026 16:15:14 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860114; bh=3WjTSvvh4sYtO98lql8YoYHvCvWGaUwBwCoDwXD+M1o=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=hkF7ch6EqHcD3DxZeXyGIrGIfxtKs5a5SjwBImofctXGh0sIrin6ePge8iHymLwEG CBXvUw0PhvfyKBZ67abkKQsDJkrTVeAPLx1tdnEoihq7pGdNJKv5o2tL6CTkiZNG0m ks73mFk9ZzLdL031rZRNmSzKMd89ZBcJiT5QHGx1rqHMmd/Z4fzjBsk0j8X9OjvVds jj24VXVgFGMsIA9BJAtH6kCnR5iFlFTBdlfMvLjtBsi/UEfhtWNU8unlBwTR1S39+P JNHx1df49oIYuOP6JHhXCGwlpVEM6oi2qUfE57wfXZEQ/euJlYUeDP15pXiXOENHsb v5CaWsxnzRamw== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 97ED7F40066; Tue, 4 Aug 2026 12:15:13 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Tue, 04 Aug 2026 12:15:13 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdfHSt+dGlSIgweIKZqd43ZIr2UVwMz+DLCV7Uxtqr2tWdjT0gNVbLG23y/ntMNr yg2u9xNueu4ZlCPesc7IGVMS6ZqisyNG2Hp2kDVIfbBL9QFp1F1pvbwsQcrNmamnsGJsZY AmMncETOYZEkhlG6T7JDS64AVGc7Ln4zXY6x3T0lHgb+OYvht9swwTIGsG7Xot0GJG9bWc fhVx9JgcfYh5ECBMh98FFx1XWl/VnjhK7S6I+ssCNmeXl+D6Do6MWM72zOvrRuLzi3YdIE +uwF6SMc7tQqt75yJIKNoQF+R4TbmXy2JAxyIWDxHsQYaXIqpYC7ilWb/r9+jjwAD2Jci2 WW/2rfk53B+6wqOwUkWkBX9tZzueTv2kVzVRGOnJreZsSVxqrC2VfDfnWJWc3kFBQBrBvj GnMlgil+/02b948U4F3se/vAZsftBNigbl8vTbnvWdDwnZBtnARN9k66R+h+WIj/cGMG32 z6558YZyvCiAUckAWMR5NSg5L/qUiMWOT65zKUtubM4T61cA21re5BJUNfIBuaHNK4uzQs szkHjosaXLoy/MvsbDxSc7cJhI5RqgqpeVf57hgZ9AQ9lKZcaC40R38llGNX4aTDr02aXh RFxm1XY4TlXQo9xEZMQDDKVkfLth3aVp7mLi+2BbP63ZjKPKdxMQfEeUi1RA X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:13 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 11/17] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Tue, 4 Aug 2026 09:14:33 -0700 Message-ID: <20260804161447.84806-12-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Arm64 already uses 64-bit preempt count and the need reschedule bit is maintained in a separate 32-bit word from the preempt count. Therefore preempt count has enough bits to represent 16 levels of NMI nesting, hence enable it for arm64. This saves a per-CPU variable and additional instructions in the NMI path. Signed-off-by: Boqun Feng --- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..349c3533cd1e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -247,6 +247,7 @@ config ARM64 select PCI_SYSCALL if PCI select POWER_RESET select POWER_SUPPLY + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 44CC64746DA for ; Tue, 4 Aug 2026 16:15:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860124; cv=none; b=hY1PuBwBMUr3tysw0QLOi+E8bSTt8MX/e6cGhF0AsoBW4VkMrpd45D8J9Y7MJZOT3WS6OG5926v3dSAGh/W07/cyOw2KwhgI4VKZ+7D6+xApxMcnHFv5eFo9PXh7q0RbxNQgQntrHt93l/lPgeDYjsWkI00yk5C+N2SS76huPJA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860124; c=relaxed/simple; bh=kLeRiC95t1IZBu689qepS3f7brPx8OxG9N0foKAWud8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=hFUO5nor2kVe89C2q/tdgF8k9e6P1GgSzGswb8/fQulpjGYGSmvRQl+rhmVDuYW4KdXYWyPG/sjq40xW3jXhlgWkAd9LBrQMfKbTCxRr30puR3dlyXCLc1hfgLXcXbwdlk8qSiUoce/TZYuv9Aqj7FSzmwxVM+zIGPGl9+Tf7hU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Tg/Vfzhx; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Tg/Vfzhx" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4A59A1F000E9; Tue, 4 Aug 2026 16:15:16 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860116; bh=fj0vODnnkGxYzHO4JCnC/iMb/acZG7slJd0L06VbQoI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Tg/VfzhxL7mOxhQWfgk135JTuAp/akopIMRdZhsjgKDODR3/3rTB1l4W7TOqvoG82 PSCaCvR7jjlJFkK9bYqAo5YE1saa5NHv+96rfVc2MgJKsHYKHA0276C+l4WDw+nCxb gwOCI68MFN5nFJ6vufIunfJjfigxWdswefJ7emqdNp9D560ty705RJ9J294I4eJwBD iG4Xb+EK4zy12O99sk2cN7OIGD5BAXu4/Cwx+AqgpXrtBSI4wnspHDv1f9OXVxihE6 I0kJzPLmoi7o7hfdm4kzjsiNc4a0SIKKR7Xggp1rw0W1VTMCi7k87tHfjgT/G7XjGz WelmnMps6TmyA== Received: from phl-compute-10.internal (phl-compute-10.internal [10.202.2.50]) by mailfauth.phl.internal (Postfix) with ESMTP id 87D75F40066; Tue, 4 Aug 2026 12:15:15 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-10.internal (MEProxy); Tue, 04 Aug 2026 12:15:15 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTFdfHSt+dGlSIgweIKZqd43ZIr2UVwMz+DLCV7Uxtqr2tWdjT0gNVbLG23y/ntMNr yg2u9xNueu4ZlCPesc7IGVMS6ZqisyNG2Hp2kDVIfbBL9QFp1F1pvbwsQcrNmamnsGJsZY AmMncETOYZEkhlG6T7JDS64AVGc7Ln4zXY6x3T0lHgb+OYvht9swwTIGsG7Xot0GJG9bWc fhVx9JgcfYh5ECBMh98FFx1XWl/VnjhK7S6I+ssCNmeXl+D6Do6MWM72zOvrRuLzi3YdIE +uwF6SMc7tQqt75yJIKNoQF+R4TbmXy2JAxyIWDxHsQYaXIqpYC7ilWb/r9+jjwAD2Jcjz 28wF4sgDT2EH23Nebbu61ApdvxCjYbSdUHRXAsdG1yTtm8Y7k8SQqW1EetYOz5cxbv2b8B W8UZFUjttuZMD52p2U81KglKeO2B5sQc+lXj5tnlucVSj4MdbyB8iFmlbFKZ9rcRiglALw Nkaqs+jWRdY5op6mEq2/TalHs5nrdXvvjdMp3r8EPU9E2Miq1TDxI6BU7o3euyEZ9hQGQV 8Jbo/QEsmRzjBOEoCuBNed3Evu1EqoMtAcIwQ8c/kEYxgbnUhFoTMR1SMX61AEE62eYnHj QnPU9CbKYNzN7FnRcxHs0qIqJWlAyOUqzJNKiMkxOTBSMSb9N3CStF4CLhpA X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:14 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Heiko Carstens Subject: [PATCH v4 12/17] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Date: Tue, 4 Aug 2026 09:14:34 -0700 Message-ID: <20260804161447.84806-13-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Heiko Carstens Convert s390's preempt_count to 64 bit, and change the preempt primitives accordingly. Signed-off-by: Heiko Carstens [boqun: Apply the corrected comment for asm block] Signed-off-by: Boqun Feng --- arch/s390/Kconfig | 1 + arch/s390/include/asm/lowcore.h | 13 +++++++--- arch/s390/include/asm/preempt.h | 43 +++++++++++++++------------------ 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..378fcd2b6181 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -273,6 +273,7 @@ config S390 select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select PCI_QUIRKS if PCI + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcor= e.h index 3b3ecc647993..5cef215d30e7 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -160,10 +160,15 @@ struct lowcore { /* SMP info area */ __u32 cpu_nr; /* 0x03a0 */ __u32 softirq_pending; /* 0x03a4 */ - __s32 preempt_count; /* 0x03a8 */ - __u32 spinlock_lockval; /* 0x03ac */ - __u32 spinlock_index; /* 0x03b0 */ - __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ + union { + struct { + __u32 need_resched; /* 0x03a8 */ + __u32 count; /* 0x03ac */ + } preempt; + __u64 preempt_count; /* 0x03a8 */ + }; + __u32 spinlock_lockval; /* 0x03b0 */ + __u32 spinlock_index; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ __u8 percpu_register; /* 0x03c0 */ __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preemp= t.h index 0a25d4648b4c..5560d5fca2a3 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -8,11 +8,8 @@ #include #include =20 -/* - * Use MSB so it is possible to read preempt_count with LLGT which - * reads the least significant 31 bits with a single instruction. - */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */ +#define PREEMPT_NEED_RESCHED 0x8000000000000000UL =20 /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -26,25 +23,25 @@ */ static __always_inline int preempt_count(void) { - unsigned long lc_preempt, count; + unsigned long lc_preempt; + int count; =20 - BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) !=3D sizeof(int)= ); - lc_preempt =3D offsetof(struct lowcore, preempt_count); - /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */ + lc_preempt =3D offsetof(struct lowcore, preempt.count); + /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED)= */ asm_inline( - ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n", - "llgt %[count],%[offalt](%%r0)\n", + ALTERNATIVE("ly %[count],%[offzero](%%r0)\n", + "ly %[count],%[offalt](%%r0)\n", ALT_FEATURE(MFEATURE_LOWCORE)) : [count] "=3Dd" (count) : [offzero] "i" (lc_preempt), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS), - "m" (((struct lowcore *)0)->preempt_count)); + "m" (((struct lowcore *)0)->preempt.count)); return count; } =20 -static __always_inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; =20 old =3D READ_ONCE(get_lowcore()->preempt_count); do { @@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc) =20 static __always_inline void set_preempt_need_resched(void) { - __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_cou= nt); } =20 static __always_inline void clear_preempt_need_resched(void) { - __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count= ); } =20 static __always_inline bool test_preempt_need_resched(void) @@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val) =20 lc_preempt =3D offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("asi %[offzero](%%r0),%[val]\n", - "asi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n", + "agsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (val), @@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val) return; } } - __atomic_add(val, &get_lowcore()->preempt_count); + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } =20 static __always_inline void __preempt_count_sub(int val) @@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_t= est(void) =20 lc_preempt =3D offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n", - "alsi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n", + "algsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "=3D@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (-1), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS)); return (cc =3D=3D 0) || (cc =3D=3D 2); #else - return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count); + return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_= count); #endif } =20 @@ -141,7 +138,7 @@ static __always_inline bool should_resched(int preempt_= offset) =20 static __always_inline int __preempt_count_add_return(int val) { - return val + __atomic_add(val, &get_lowcore()->preempt_count); + return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } =20 static __always_inline int __preempt_count_sub_return(int val) --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4545A4749C9; Tue, 4 Aug 2026 16:15:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860124; cv=none; b=XShtnKHknGXjj1FpT9o1oaMPspRy1CvWJszCs7zp/3Ds60d6nCoJE1XdETMG4NHHo+Z8BgD2muMd4y2Snex/e5h2ID2sRBcpP/In9DcX5SETrxr7eCI6ZPsA0d9be89smOxqGN/TFfWaO0rN1bu4WsVkSIum2tJxMawHbGLN7E4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860124; c=relaxed/simple; bh=PwsYruj68uHekcgGg2+D0S/5Y1JU0FvXwh9yngRw4wU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=a6KtJusftZTANeSzpEOF2CPF1pCR7/xjKDU8usNN90SoPKpEGw4ag27i1yFJ75PZK7o2i0YgQQqRZC3lKlFGKf0FqoOBPmS4o46O/eVNDZl/AMXaTS3loe6HyNUWGWjRqWSPIiWsJFPW6rU97DnMgamUrGnusLIFuxBy+v5u0NE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=UaJt/orZ; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="UaJt/orZ" Received: by smtp.kernel.org (Postfix) with ESMTPSA id C4ADB1F00ACA; Tue, 4 Aug 2026 16:15:17 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860118; bh=c1gjxyEZxiWV+lR91hdPXb5LqwKRvifjfGHEayyotg8=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=UaJt/orZq8aoIbSdAceGioNxglD9TvN90gVLFeGrxXURe4akIozM89u3Jj+hPL/B9 +X9q1jjfzODB6RJwTXGIviIwgruIKMnzsjtL5zxvfbU5sDpSgN5uvS3e7kiyTUI6if Ihlgedw2Nv/T6FV1riPhL3rmyQIb3yAzkBkfgb+42Tj/cZexLiHgNW/8POdqFTWlYV W6/uWKI3tVEzoWzKRVAV0F7jC19Qch8FhLpuH0EWxCLeWzc0dcQXsXsYV0vx2zeG1m WqLNLgSRfFnshsgnGixCqSVuR0EVwgTYVtVcVd2BfrlH+IbsZX/H5HpaOwHdTxA/us 4qTe/ns4S1krg== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id 0D043F40066; Tue, 4 Aug 2026 12:15:17 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Tue, 04 Aug 2026 12:15:17 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGWpEpVyD1Wq7sf3aa73PVngtFiTOqR7nSVa+oR1a+xJBvuLB19FlSyZiPSqaGnqf mTcGiPEwbNDRyzULygS0C0KFc2nTddz/nikK5FiLrHcDr9T83+robkxqRQWCb8N/L7hnkX N5/7H2+B643pHh/o1C9v0il+2uw03O85e9l8+KDALGBfzRohermLGx6emmOwP2LXFvgKKY OqlnPWwi5FJDcrW9D/xIuH5dIUKdZj4Dkwn9DeYZtiLj0I9mtPOvnue5AmaiYISqY27LEo va8TSK5sGOUc1Q5JONq34xK6ShiWmU0CZpEvIgBamhBWd0KAMVNFKCaRdTjjwvXc8L2HCq ST+ShJJ5oOjOAXxpyK0fgEOIn+q3Q8nRPbrv0C6v6xorcBVX0/TbibmXLSbPioHwSLMq2/ n4jU3T8iR+fqJefx8MMIZIwE39lUUBuYdvOE1+vq01XhnzkT49Ji6LVY3hnjVDhUDBJMZq PdWjDyyJZsfCfr5Dtbn6ruLHld1VI4RmbFwx+s3BTb+50COli+nwbKxLjo1S1VnS694Bc2 i3hfAWWkiP0VTz1VDMhY8DkurjQK6CWiRMgkie+QXjnKHQ9w71ellyOm/LVSjlOIo4FDHd 7cVyqR2U3q14ByPO2ruCO3Z5KU2dC3tvLVRV0R3iOQIkMI4QFT9pDJhxhNag X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:16 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Benno Lossin , Andreas Hindborg Subject: [PATCH v4 13/17] rust: Introduce interrupt module Date: Tue, 4 Aug 2026 09:14:35 -0700 Message-ID: <20260804161447.84806-14-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Lyude Paul This introduces a module for dealing with interrupt-disabled contexts, including the ability to enable and disable interrupts along with the ability to annotate functions as expecting that IRQs are already disabled on the local CPU. Signed-off-by: Lyude Paul Reviewed-by: Benno Lossin Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Signed-off-by: Boqun Feng --- rust/helpers/helpers.c | 1 + rust/helpers/interrupt.c | 18 ++++++++ rust/helpers/sync.c | 5 +++ rust/kernel/interrupt.rs | 89 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 5 files changed, 114 insertions(+) create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/kernel/interrupt.rs diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..0d85b5e68ec2 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -65,6 +65,7 @@ #include "irq.c" #include "fs.c" #include "gpu.c" +#include "interrupt.c" #include "io.c" #include "jump_label.c" #include "kunit.c" diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c new file mode 100644 index 000000000000..51b319bd4c00 --- /dev/null +++ b/rust/helpers/interrupt.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ + local_interrupt_disable(); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ + local_interrupt_enable(); +} + +__rust_helper bool rust_helper_irqs_disabled(void) +{ + return irqs_disabled(); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c index 82d6aff73b04..4f474fe847c4 100644 --- a/rust/helpers/sync.c +++ b/rust/helpers/sync.c @@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(str= uct lock_class_key *k) { lockdep_unregister_key(k); } + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ + lockdep_assert_irqs_disabled(); +} diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs new file mode 100644 index 000000000000..a880ec3b8538 --- /dev/null +++ b/rust/kernel/interrupt.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Interrupt controls +//! +//! This module allows Rust code to annotate areas of code where local pro= cessor interrupts should +//! be disabled, along with actually disabling local processor interrupts. +//! +//! # =E2=9A=A0=EF=B8=8F Warning! =E2=9A=A0=EF=B8=8F +//! +//! The usage of this module can be more complicated than meets the eye, e= specially surrounding +//! [preemptible kernels]. It's recommended to take care when using the fu= nctions and types defined +//! here and familiarize yourself with the various documentation we have b= efore using them, along +//! with the various documents we link to here. +//! +//! # Reading material +//! +//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/52= 0076) +//! +//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/= preempt-locking.html + +use crate::types::NotThreadSafe; + +/// A guard that represents local processor interrupt disablement on preem= ptible kernels. +/// +/// [`LocalInterruptDisabled`] is a guard type that represents that local = processor interrupts have +/// been disabled on a preemptible kernel. +/// +/// Certain functions take an immutable reference of [`LocalInterruptDisab= led`] in order to require +/// that they may only be run in local-interrupt-disabled contexts on pree= mptible kernels. +/// +/// This is a marker type; it has no size, and is simply used as a compile= -time guarantee that local +/// processor interrupts are disabled on preemptible kernels. Note that no= guarantees about the +/// state of interrupts are made by this type on non-preemptible kernels. +/// +/// # Invariants +/// +/// Local processor interrupts are disabled on preemptible kernels for as = long as an object of this +/// type exists. +pub struct LocalInterruptDisabled(NotThreadSafe); + +/// Disable local processor interrupts on a preemptible kernel. +/// +/// This function disables local processor interrupts on a preemptible ker= nel, and returns a +/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible = kernels, this function is +/// a no-op. +/// +/// **Usage of this function is discouraged** unless you are absolutely su= re you know what you are +/// doing, as kernel interfaces for Rust that deal with interrupt state wi= ll typically handle local +/// processor interrupt state management on their own and managing this by= hand is quite error +/// prone. +#[inline] +pub fn local_interrupt_disable() -> LocalInterruptDisabled { + // SAFETY: It's always safe to call `local_interrupt_disable()`. + unsafe { bindings::local_interrupt_disable() }; + + LocalInterruptDisabled(NotThreadSafe) +} + +impl Drop for LocalInterruptDisabled { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariants, a `local_interrupt_disable()` must= be called to create this + // object, hence calling the corresponding `local_interrupt_enable= ()` is safe. + unsafe { bindings::local_interrupt_enable() }; + } +} + +impl LocalInterruptDisabled { + /// Assume that local processor interrupts are disabled on preemptible= kernels. + /// + /// This can be used for annotating code that is known to be run in co= ntexts where local + /// processor interrupts are disabled on preemptible kernels. It makes= no changes to the local + /// interrupt state on its own. + /// + /// # Safety + /// + /// For the whole life `'a`, local interrupts must be disabled on pree= mptible kernels. This + /// could be a context like, for example, an interrupt handler. + #[inline] + pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled { + const ASSUME_DISABLED: &LocalInterruptDisabled =3D &LocalInterrupt= Disabled(NotThreadSafe); + + // Confirm they're actually disabled if lockdep is available + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled(= )`. + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + ASSUME_DISABLED + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9512af7156df..2ee6c24d39c2 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -82,6 +82,7 @@ pub mod impl_flags; pub mod init; pub mod interop; +pub mod interrupt; pub mod io; pub mod ioctl; pub mod iommu; --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C883A4749DA; Tue, 4 Aug 2026 16:15:19 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860126; cv=none; b=j1s2TGvNcd27bpeyRqiZjYIC5X+7iIuQbkNU29eD9orstX46UMDNnW2WKOWi3AbgJ1RyHpe4dhmH7m4pi1rVpWCJr8HSLsFHvXJQ52R8TM5k3FuiYBS85VasDWSte13VIeBoRI1sDIs9zH/mcWOdd3w6GCx1Pm56E8WE9/4SFac= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860126; c=relaxed/simple; bh=F7Y7A/nUkO50uzCj7ZdwiNWURz04OlYPwgBgKCWJjsA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=mD2xjd16bKWnn7rDNcBxO0U7plD0NPMq04ALg4VnJQKWTs3kzpaspALeDGVkVa3rfyIwG/hg5xf9Nr06NscI9A0GroUfkCIPhSiDpCE78Yo47C0Hwbbg0hu1e4DZQ8NjAYvlrYjVTBAFvVXnUT0sLZ9bNx8RV68YG40/MBns4Gs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=KYFWZjyl; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="KYFWZjyl" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3C7A41F00A3E; Tue, 4 Aug 2026 16:15:19 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860119; bh=36qEnnu+VHFxkCyChMu3DL672ehrfvpUeGwqcHC+BuQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=KYFWZjylIU206KPJQHAouWrO2zaoUAK27PgJtqD1nt9WFANanrgO9fhZSmWoKBaLo wrqcoCVfWeyW7XLpUdK3jD4jvwUxYDrsSqDEwtSMrt3xieKVcdWgD/hPm8R+HtfGog hwQ1yK7wUuPTufKocA+lmwMV+p57vkFa4bGdEoq4giev536YO11AgG6odSxz+0knGZ kTGNAttDinyQOJTX/5iNgalCnnPN5NISsGSeak1lHDlObMRixqQIQzLDWD3o0f6PgT /AGH09Xtp8QOrgCSH2vTZrMeBNZNUGxp/bbDH2CFSDZ/foziDVVDS0eEQ9H5+6l+SN xK5+4f0lCcnhw== Received: from phl-compute-01.internal (phl-compute-01.internal [10.202.2.41]) by mailfauth.phl.internal (Postfix) with ESMTP id 7810CF40067; Tue, 4 Aug 2026 12:15:18 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-01.internal (MEProxy); Tue, 04 Aug 2026 12:15:18 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbXI DriopieP6f4X5f5dlxaM/ucjKy5utE+zF0SNby58ScV/dtasTk8xN9P0Qztpku5lS5RjAV sqzv7ML67P+yf/Wqz5iSuX+nRwqvrW5Uyh9RWCv30nllzYNVl+I/uWvPwTKDsB582AN+90 Nsk1NWWF3Eri+QGYJC8hQqtSFkn03UPOdXGNbrK1P4WsBtVOn0mGhw7OjcwD2NwaZ2cyng MIJPyTYGbvcuWfgM2L7d5U0NxhFS8ctpq3qYsWu+6uAlVRbAYpFXOyWR6C7N7wO8Luf9JA IyjfOd7r8Ve2T69ogVZduBtxD8YcimcgtJa1CFzAojZNLtpnp8sIt10UxxNw X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:17 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, Andreas Hindborg Subject: [PATCH v4 14/17] rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers Date: Tue, 4 Aug 2026 09:14:36 -0700 Message-ID: <20260804161447.84806-15-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" spin_lock_irq_disable() and spin_unlock_irq_enable() are inline functions, to use them in Rust helpers are introduced. This is for interrupt disabling lock abstraction in Rust. Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Signed-off-by: Boqun Feng --- rust/helpers/spinlock.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 4d13062cf253..d53400c15022 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinl= ock_t *lock) { lockdep_assert_held(lock); } + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ + spin_lock_irq_disable(lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ + spin_unlock_irq_enable(lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ + return spin_trylock_irq_disable(lock); +} --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BDDC9476CF9 for ; Tue, 4 Aug 2026 16:15:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860130; cv=none; b=iHKIhIv3S49Sa3g/AtQXUOk5qrM9R697nIQ72Me2jgxNP8yXzU1OUpZGhvfrdlYPP5D6kwwwWhbDEpv6pbtHvhktYYLekC2K1XTi/mESBPx4TEZIm6jzFP7Chjzg3P0pBg4ViRNgBOWXugWtJq8Dq2R/Z/1DB4AWr0WF4H6cpRs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860130; c=relaxed/simple; bh=1B574WZxr85tEXgvJ/lsuCd24hw6r0EvD8jaMRrm6yk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=dAs09fBe046uEE9WnfV/GzJRSQ/ePVJZSPuZCgpTFxtCLJi6+wr++RoFOEjHE24e932KiT71baQ/IYoaa4lyR8m2M2NttX9ZMuyQ2ps8kf+/+kzXHJcdDZjY6kEqkQcoolbglPzuu+4k21p2kldrwVHJZc8EdHP/rh2q7t/REXI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=k5pJ+ckn; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="k5pJ+ckn" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B079C1F00ACF; Tue, 4 Aug 2026 16:15:20 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860121; bh=Ehxr5VUxk0jCrAQvRW8oaPBfKD8Ta2YHENyPPAE1YHc=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=k5pJ+cknPV9guqpPjGzLh1qVLzyCz9aJ4bRQXRfhzZlJKC1whKvxDZmkadHvZ6Vw5 7lIIgNZ1W11597tOYdqA9sUoibfcX9dhRLWnrRZAJlpkGYQMfxJYvWelHZbq72D9hf tvXvMw7M7lIAZY0U+mFaY0kklFp64Q9crJw25/R7XDSxQkJ5zEjPqjzr5H2PEdJKsr BkylBhHbic9cr0lFnEZdEPmq00SyoBq7R6pBj6S9gSE5pHeJSyT6/g2G7h7ZUxi4ed 6Su8oJnUKUVMbqoJt+ZFSrFLHupM7c4ieZz74R5h19ve7+8Pf2p/NPoQuk0nxo4MKQ bw/mUtm34D5kw== Received: from phl-compute-01.internal (phl-compute-01.internal [10.202.2.41]) by mailfauth.phl.internal (Postfix) with ESMTP id ECC1BF40066; Tue, 4 Aug 2026 12:15:19 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-01.internal (MEProxy); Tue, 04 Aug 2026 12:15:19 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTEjNALs/vHMOloma/FioWgq38LEDphqLbi/MDKjZ+skyek9CBm72qNR6fF0uZPdGp TNcnAR4AQwTabPyqXb4b8VVQVFCNoT48fsLJV/4jNEVJ0A9eFv0ovpeauV2dgztP0aY1Y0 jIaZAzpZKJmSr5bjx5yubrqLJ++dH6uDgWLI4XXVkSa8jJqAXQgEQlXOv7WGccuHPTRlER Ylu8rJket0jbyQhGWZ+weYiCIeD4wr9LDagDzAHrKU27qzYbzXZIXOYWeFfQKFS5QVbMSU NlfDPJ2QDHkDrdBA1iUAq2dsmA2mOrsNEsIEQsqJKBWsANZHLO8XMG8BasrOULdBqAdbPC 0Y+rycPLPuV1FKoKiCpkBeZSEDMwM7l1MZM+HjJRJVWyduAfzslhvebXdopARms4VNgoxb VPxqS/ibVICmklGBL3QsyYGEjYTNyM6TXqUiAroTBD0FISvOhCS+BPZfMUUO+28SL+aMh2 /gyC5bMHNE+nFrG4uqq9LACee3tcCp9f9b2COVhAYtBGEFTumjY13sXpoRqh0kPxNs12ir gA7kU0UWLYMS7sfoIKzFOdyMTjnWhRJylElsFocEOYyyzYy9n8pU3EfQrUsRJEvL7/Lo6h lT2Vk/Dvtkj1kyygKhL2b+cuxIA90gAwQrkbfaIw8PVvTDWS9Zk3UIJUNZ5A X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:19 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 15/17] rust: sync: Use super::* in spinlock.rs Date: Tue, 4 Aug 2026 09:14:37 -0700 Message-ID: <20260804161447.84806-16-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul No functional changes. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng --- rust/kernel/sync/lock/spinlock.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index ef76fa07ca3a..d75af32218ba 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -3,6 +3,7 @@ //! A kernel spinlock. //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use super::*; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -82,7 +83,7 @@ macro_rules! new_spinlock { /// ``` /// /// [`spinlock_t`]: srctree/include/linux/spinlock.h -pub type SpinLock =3D super::Lock; +pub type SpinLock =3D Lock; =20 /// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend; @@ -91,13 +92,11 @@ macro_rules! new_spinlock { /// /// This is simply a type alias for a [`Guard`] returned from locking a [`= SpinLock`]. It will unlock /// the [`SpinLock`] upon being dropped. -/// -/// [`Guard`]: super::Guard -pub type SpinLockGuard<'a, T> =3D super::Guard<'a, T, SpinLockBackend>; +pub type SpinLockGuard<'a, T> =3D Guard<'a, T, SpinLockBackend>; =20 // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclus= ion. `relock` uses the // default implementation that always calls the same locking method. -unsafe impl super::Backend for SpinLockBackend { +unsafe impl Backend for SpinLockBackend { type State =3D bindings::spinlock_t; type GuardState =3D (); =20 --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E9BEF473C90 for ; Tue, 4 Aug 2026 16:15:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860134; cv=none; b=t+TMnOe3iheYWWmZ3RJrAQav9GzVkyX3h6YD9dGzSkkVg35sM1nxcw1fuu+4M6E5Asc9kYWhftHJuk2uHcB2Ch3MEyxuGTC4jkq8JfdSFGq6mTXoZ0TmV/Yyul8m4gU5R6zgbpGUVQ5xiUXr/qDdPEbLtINA6h5DV5fIhpq02Wc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860134; c=relaxed/simple; bh=kZU5uFqdjWiRbrVf1fTNRv9ePx+qKQEq8Hc6Bs6H2Cs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tIRAJ59agaIogyMQGfHN8m+oqednw6/TT+AMmn78IoAx0IuWLF0TFyQth1wxHjXehmDvkpow3VztzrD5tlHrKBMM2uC3c8peVzM9019xYyykhScRoVK6cETNtA+FImV8J2rroO4d/5FHxLVHg92CgFWycLEd3nVvbmLY86lvxV8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=BWEzvKpr; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="BWEzvKpr" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 5CECF1F00ADB; Tue, 4 Aug 2026 16:15:22 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860122; bh=2MYZgIP+osziR2bs888gmmBvgXIfwDVrZfFhcS+axXE=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=BWEzvKprIWof6jEJHHyNb4i8vHe/Slyz+CGATfaa2/ueOxjhQuivs47yKwukEnBi5 o5a83COJ15flU+J/pRTw+jC9cDGQ1f+CB8f3O0WJkjjve4RxLaIDREQzZ8iJzjGIyw tcp/VZEnO5cpOSEaAZeUN1WSsuVIyMGRwq4b8LQHeMEtfiSqCBDJ5a0y5SUEEUoFLg lWFbtNLPr38HObFQjQgUXCBi214JQN9ykqtRF1NodLsCXGUHm51lrEQFXtIIiDUPcW BCVMXDdGalNlkm98xr999nl65NrxkFt0r8BO4JFpd3/RewKkDLODkYGg1TDndBWkQE 1RjYLxm7p24Bg== Received: from phl-compute-03.internal (phl-compute-03.internal [10.202.2.43]) by mailfauth.phl.internal (Postfix) with ESMTP id 9AD19F40066; Tue, 4 Aug 2026 12:15:21 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-03.internal (MEProxy); Tue, 04 Aug 2026 12:15:21 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTGieA2PtfV9jSFchkBSUGRvAWb1NITTGgeikxrU+GWeOukLWTSA/NzmsBjdHwUYbe 0erteAkfc4UaRDf0dMOdxgNSF+x3Zu0BbdfmeM7AKPbfK0OPT0HLroc+sluO/cC9IMMPSf UC/oH0bN/7dDLlq3yKu6wzn2cf4YXKbKDiff65imTlvzFYVAi2jiR3u4/PsSuR9+0G3TT4 iRoMH87H0urJqIjMbAilmNI/FWk0WY51GTR9tXwGcnj+pTvT++vrrx98cMKjuNNccHsv3y mSjasO/eyTPpxx12FEV6IRjl72E/fAhMzSnlHo2hlR954mzd3YIM5xarphE6YabMo8hGcY oMCHhIEUbq4hK8080vy7dQkB1j0jHmE0hq1BwRdoXk3RLMqAcOFs5PsgB1ALvDUw+HJB5r uljmvzYM9jrhHULSya9H4yoYhTC/m/AKFxlOrGsF75H6KAYOlMNeBygl2hVkHd8rflL/19 RzJ8SpgRGesVWVf57MJDth0r0nlU89BCC0lmI0GShYRWqof3L2F0YK2e0F1BRv6WtiEvbD EkDHaWcYtxzZmCZ16PnAoCA7tVNjay+5brMnQEdzHYy/p1JIof4gdd+TxQ5s4kj8T3xqs4 XmWO2J8oye8LTSVV8RZXYim9vk3cY1Z7Hux0ZEul1KjIuuLT7nuPCjgk+p8Q X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:20 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 16/17] rust: sync: Add SpinLockIrq Date: Tue, 4 Aug 2026 09:14:38 -0700 Message-ID: <20260804161447.84806-17-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul A variant of `SpinLock` that ensures interrupts are disabled in the critical section. `lock()` will ensure that either interrupts are already disabled or disable them. `unlock()` will reverse the respective operation. [Boqun: Port to use spin_lock_irq_disable() and spin_unlock_irq_enable()] Signed-off-by: Lyude Paul Reviewed-by: Gary Guo Signed-off-by: Boqun Feng --- rust/kernel/sync.rs | 9 +- rust/kernel/sync/lock/global.rs | 3 + rust/kernel/sync/lock/spinlock.rs | 230 ++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..df4f2604ff9b 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -27,7 +27,14 @@ pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBac= kend, GlobalLockedBy}; pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; -pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, + SpinLock, + SpinLockGuard, + SpinLockIrq, + SpinLockIrqGuard, // +}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global= .rs index ec2dd84316fc..ebb10521d8bd 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -306,4 +306,7 @@ macro_rules! global_lock_inner { (backend SpinLock) =3D> { $crate::sync::lock::spinlock::SpinLockBackend }; + (backend SpinLockIrq) =3D> { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index d75af32218ba..872544948e5d 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,6 +4,7 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. use super::*; +use crate::prelude::*; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -143,3 +144,232 @@ unsafe fn assert_is_held(ptr: *mut Self::State) { unsafe { bindings::spin_assert_is_held(ptr) } } } + +/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-= created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on = the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock_irq { + ($inner:expr $(, $name:literal)? $(,)?) =3D> { + $crate::sync::SpinLockIrq::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock= _class!()) + }; +} +pub use new_spinlock_irq; + +/// A variant of `SpinLock` that ensures interrupts are disabled in the cr= itical section. +/// +/// For more info on spinlocks, see [`SpinLock`]. For more information on = interrupts, +/// [see the interrupt module](kernel::interrupt). +/// +/// # Examples +/// +/// The following example shows how to declare, allocate initialise and ac= cess a struct (`Example`) +/// that contains an inner struct (`Inner`) that is protected by a spinloc= k that requires local +/// processor interrupts to be disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// c: SpinLockIrq, +/// #[pin] +/// d: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }), +/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example` +/// let e =3D KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// +/// // Accessing an `Example` from a context where interrupts may not be d= isabled already. +/// let c_guard =3D e.c.lock(); // interrupts are disabled now, +1 interru= pt disable refcount +/// let d_guard =3D e.d.lock(); // no interrupt state change, +1 interrupt= disable refcount +/// +/// assert_eq!(c_guard.a, 0); +/// assert_eq!(c_guard.b, 10); +/// assert_eq!(d_guard.a, 20); +/// assert_eq!(d_guard.b, 30); +/// +/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just = yet, since d_guard is +/// // still in scope. +/// drop(d_guard); // Last interrupt disable reference dropped here, so in= terrupts are re-enabled +/// // now +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrq =3D super::Lock; + +/// A kernel `spinlock_t` lock backend that can only be acquired in interr= upt disabled contexts. +pub struct SpinLockIrqBackend; + +/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`= SpinLockIrq`] using +/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local= processor's interrupt +/// disablement refcount upon being dropped. +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrqGuard<'a, T> =3D Guard<'a, T, SpinLockIrqBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclus= ion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockIrqBackend { + type State =3D bindings::spinlock_t; + type GuardState =3D (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for = writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `p= tr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock_irq_disable(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardStat= e) { + // SAFETY: The safety requirements of this function ensure that `p= tr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock_irq_enable(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initial= ized before use. + let result =3D unsafe { bindings::spin_trylock_irq_disable(ptr) }; + + if result !=3D 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initial= ized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +#[kunit_tests(rust_spinlock_irq_condvar)] +mod tests { + use super::*; + use crate::{ + sync::*, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, + }; + + struct TestState { + value: u32, + waiter_ready: bool, + } + + #[pin_data] + struct Test { + #[pin] + state: SpinLockIrq, + + #[pin] + state_changed: CondVar, + + #[pin] + waiter_state_changed: CondVar, + + #[pin] + wait_work: Work, + } + + impl_has_work! { + impl HasWork for Test { self.wait_work } + } + + impl Test { + pub(crate) fn new() -> Result> { + Arc::try_pin_init( + try_pin_init!( + Self { + state <- new_spinlock_irq!(TestState { + value: 1, + waiter_ready: false + }), + state_changed <- new_condvar!(), + waiter_state_changed <- new_condvar!(), + wait_work <- new_work!("IrqCondvarTest::wait_work") + } + ), + GFP_KERNEL, + ) + } + } + + impl WorkItem for Test { + type Pointer =3D Arc; + + fn run(this: Arc) { + // Wait for the test to be ready to wait for us + let mut state =3D this.state.lock(); + + // Make sure the interrupts actually turned off + // SAFETY: It's always safe to call `lockdep_assert_irqs_disab= led()` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + while !state.waiter_ready { + this.waiter_state_changed.wait(&mut state); + } + + // Deliver the exciting value update our test has been waiting= for + state.value +=3D 1; + this.state_changed.notify_sync(); + } + } + + #[test] + fn spinlock_irq_condvar() -> Result { + let testdata =3D Test::new()?; + + let _ =3D workqueue::system().enqueue(testdata.clone()); + + // Let the updater know when we're ready to wait + let mut state =3D testdata.state.lock(); + state.waiter_ready =3D true; + testdata.waiter_state_changed.notify_sync(); + + // Wait for the exciting value update + testdata.state_changed.wait(&mut state); + assert_eq!(state.value, 2); + Ok(()) + } +} --=20 2.50.1 (Apple Git-155) From nobody Fri Oct 2 06:16:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C96B54749F4 for ; Tue, 4 Aug 2026 16:15:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860135; cv=none; b=M+TARQtTo59mm86bvCb++eAzwdG0fiuMGRPm55zphNOtlF+xhVjhC/mWTIadbF948roOVk9djL0gqHUHQaxxeyCEJx1ohYPEKe33K/erMsynGn/BHyKUO1Q66LMTJl+67W8XvevxNTrjDUXDDB+6S577NfiBcaRWDTBT0sMEbVw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1785860135; c=relaxed/simple; bh=yOW+vEyMmlLd/rvdAQOpfI7rRLssPojgy8OC8Xdb7b8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=MpmjQePwnAfn+Y60BPRMGQopyur78P9vXQgX5hAZx1MxzU0GarvjGleI5PAvge5+K7XQtJCR3O0BrdIYm9Vcp3oF4R6XIlAUoSO/XpOuArZ4e5RcBo5CS6XS38oL45PgTYB/EAV8GTCbai7Q/Km+H92G/gyLqmLDVtSWo0xG+so= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ZTE4IoRU; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ZTE4IoRU" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A34321F00ADF; Tue, 4 Aug 2026 16:15:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1785860125; bh=J5r9F5NVhPBA6Gj8qaTBPl3yuPTDh/upo+Mtrw3vN6s=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ZTE4IoRUwljH2Jdg8+P9BzELJXYRNMj/l/KDW4gllVlVijwpfQIXiO21eHMeW3QPL 3JlF+P6+V5OiHsoNIXi/Gqi8Ekc9VqfsvBKl1q//3mbVIQDPQuRHN/il5xhASL3UwS H5xI+yA5sYwNXKsIbLb6SqC19rTol4ZZJDfSBwjiktkQesn4eQCuXQOYqSVnu6/5uV 88o0KAp2+AaV1t/2c9aXbhndYt73wqR/xlWKfZ0XTrUm6oXErCE7fni4FOPfux4E0a 5JsxxDtoDii3/Rizm4t/C1Rvaw2x7VSVeNKmz47SxYXp4sH/N+Yj0C5ZQw39fHNkCB ZE9CGrz3BxPMQ== Received: from phl-compute-05.internal (phl-compute-05.internal [10.202.2.45]) by mailfauth.phl.internal (Postfix) with ESMTP id E0698F40068; Tue, 4 Aug 2026 12:15:23 -0400 (EDT) Received: from phl-frontend-03 ([10.202.2.162]) by phl-compute-05.internal (MEProxy); Tue, 04 Aug 2026 12:15:23 -0400 X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: dmFkZTE6f1b/PLD7WfUEjv3UPH0IuooDzdONFOoQTajOMPpDZsgOHZQOVqJ4v+RHATO1sj EZSfNibFX853su8vPqAqnUSuvHiUbplQY0egsm0mTGT1lvu1S/5Nh/tuqaumzTAbEEqn88 A4HuRrhsSYsTxma98RYyzM2Gbo4E2wg96qbDSuNUia/kyZtd9PuUf9ikEEX+FWBaYhk2LZ 4A6T9U94iL3j6tNfmRpUMp/Z8p6+0GWHZbXf2pAo98Kg1ESY7btybSSSUnrcHypkA1CoVm T+WboQPfBLNhjm5E0DtWi31bTX/atiaNKg8ICM8aktklu407EF+Nx0w3ySFDe9rX99/5c7 VHxd0IBeRLjSglLmGGu4qcFo6l3dFM8dlqsmahN6miVS0lDN1uZWPAhgUWk58n5dNfTejY O3XlDdY3zr/a45OAr6aLsOyn9N9frGmsOzWcXGIDsrsKwxBBbBdBSHbc7U+/PXIzYEdnIp XLi0Fm5TYkwEKFgsBYim6ZlA6G3VRnEbfq+bx4qUJtD9jjD+UHYqCmFNGzXeZZUwbGHCjb 0FrfHhihkhv4i0JVaWFshEQPJ8wWTnJhu59no68G7UkSd/B0+bGTmdVG/7sYc/sGzR0Guq kHc5SXgxoWcLPZfgI2Ldw1o/s1T7ccCNIeshuUsOQ+f6gYWEB4R/m+m9jv2g X-ME-Proxy: Feedback-ID: i8dbe485b:Fastmail Received: by mail.messagingengine.com (Postfix) with ESMTPA; Tue, 4 Aug 2026 12:15:23 -0400 (EDT) From: Boqun Feng To: Peter Zijlstra Cc: "Ingo Molnar" , "Will Deacon" , "Boqun Feng" , "Waiman Long" , "Gary Guo" , "Alice Ryhl" , "Lyude Paul" , "Daniel Almeida" , =?UTF-8?q?Onur=20=C3=96zkan?= , "Miguel Ojeda" , "Danilo Krummrich" , linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org Subject: [PATCH v4 17/17] rust: sync: Introduce SpinLockIrq::lock_with() and friends Date: Tue, 4 Aug 2026 09:14:39 -0700 Message-ID: <20260804161447.84806-18-boqun@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260804161447.84806-1-boqun@kernel.org> References: <20260804161447.84806-1-boqun@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Lyude Paul `SpinLockIrq` and `SpinLock` use the exact same underlying C structure, with the only real difference being that the former uses the irq_disable() and irq_enable() variants for locking/unlocking. These variants can introduce some minor overhead in contexts where we already know that local processor interrupts are disabled, and as such we want a way to be able to skip modifying processor interrupt state in said contexts in order to avoid some overhead - just like the current C API allows us to do. In order to do this, we add some special functions for SpinLockIrq: lock_with() and try_lock_with(), which allow acquiring the lock without changing the interrupt state - as long as the caller can provide a LocalInterruptDisabled reference to prove that local processor interrupts have been disabled. In some hacked-together benchmarks we ran, most of the time this did actually seem to lead to a noticeable difference in overhead: From an aarch64 VM running on a MacBook M4: lock() when irq is disabled, 100 times cost Delta { nanos: 500 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 292 } lock() when irq is enabled, 100 times cost Delta { nanos: 834 } lock() when irq is disabled, 100 times cost Delta { nanos: 459 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 291 } lock() when irq is enabled, 100 times cost Delta { nanos: 709 } From an x86_64 VM (qemu/kvm) running on a i7-13700H lock() when irq is disabled, 100 times cost Delta { nanos: 1002 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 729 } lock() when irq is enabled, 100 times cost Delta { nanos: 1516 } lock() when irq is disabled, 100 times cost Delta { nanos: 754 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 966 } lock() when irq is enabled, 100 times cost Delta { nanos: 1227 } (note that there were some runs on x86_64 where lock() on irq disabled vs. lock_with() on irq disabled had equivalent benchmarks, but it very much appeared to be a minority of test runs.) While it's not clear how this affects real-world workloads yet, let's add this for the time being so we can find out. This makes it so that a `SpinLockIrq` will work like a `SpinLock` if interrupts are disabled. So a function: (&'a SpinLockIrq, &'a LocalInterruptDisabled) -> Guard<'a, .., Spin= LockBackend> makes sense. Note that due to `Guard` and `LocalInterruptDisabled` having the same lifetime, interrupts cannot be enabled while the Guard exists. Signed-off-by: Lyude Paul Reviewed-by: Gary Guo Signed-off-by: Boqun Feng --- rust/kernel/sync/lock/spinlock.rs | 92 ++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spin= lock.rs index 872544948e5d..aafc80125f59 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,7 +4,10 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. use super::*; -use crate::prelude::*; +use crate::{ + interrupt::LocalInterruptDisabled, + prelude::*, // +}; =20 /// Creates a [`SpinLock`] initialiser with the given name and a newly-cre= ated lock class. /// @@ -160,6 +163,16 @@ macro_rules! new_spinlock_irq { =20 /// A variant of `SpinLock` that ensures interrupts are disabled in the cr= itical section. /// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bind= ings will modify the +/// interrupt state to ensure that local processor interrupts remain dis= abled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`]= token is present and +/// local processor interrupts are already known to be disabled, in whic= h case the local +/// interrupt state will not be touched. This method should be preferred= if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// /// For more info on spinlocks, see [`SpinLock`]. For more information on = interrupts, /// [see the interrupt module](kernel::interrupt). /// @@ -213,7 +226,47 @@ macro_rules! new_spinlock_irq { /// # Ok::<(), Error>(()) /// ``` /// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_w= ith()`] in a function +/// which can only be called when local processor interrupts are already d= isabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in n= o-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled= ) { +/// // Because we know interrupts are disabled from interrupt_disable,= we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e =3D KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard =3D local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// /// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with pub type SpinLockIrq =3D super::Lock; =20 /// A kernel `spinlock_t` lock backend that can only be acquired in interr= upt disabled contexts. @@ -278,6 +331,43 @@ unsafe fn assert_is_held(ptr: *mut Self::State) { } } =20 +impl Lock { + /// Casts the lock as a `Lock`. + #[inline] + fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisa= bled) -> &'a SpinLock { + // SAFETY: + // - `Lock` and `Lock` = both have identical data + // layouts. + // - As long as local interrupts are disabled (which is proven to = be true by _context), it + // is safe to treat a lock with SpinLockIrqBackend as a SpinLock= Backend lock. + unsafe { core::mem::transmute(self) } + } + + /// Acquires the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock= ::lock()`] function when + /// possible for [`SpinLockIrq`] locks. + #[inline] + pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) ->= SpinLockGuard<'a, T> { + self.as_lock_in_interrupt(context).lock() + } + + /// Tries to acquire the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock= ::try_lock()`] function + /// when possible for [`SpinLockIrq`] locks. + /// + /// Returns a guard that can be used to access the data protected by t= he lock if successful. + #[must_use =3D "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock_with<'a>( + &'a self, + context: &'a LocalInterruptDisabled, + ) -> Option> { + self.as_lock_in_interrupt(context).try_lock() + } +} + #[kunit_tests(rust_spinlock_irq_condvar)] mod tests { use super::*; --=20 2.50.1 (Apple Git-155)