From nobody Mon Feb 9 04:27:44 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CAB8C2BDC15; Mon, 3 Nov 2025 07:19:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154353; cv=none; b=EgooI0OwDCPfWrJWYPq1HmUskypYz8/V0xcJRJJP1rR5vw8XnHFUfFspQT101vGSmEj8LorzTv2g8plm3JPQUwH3Mghng7QGNk4s9RCxud38MNgt736ef+oxEF5MIGm01j87TcC/Pg2OggBHAeJwdkae32cs+cnhYAOtpXhxeJI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154353; c=relaxed/simple; bh=wzU2Qnm2GHt6DLOx5HoozrHwNMWUtEvwOpzlAH8V7/A=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=Jn3xUt6bGrbZYx8q6WfkAf+6Zn1advPiQjf9oq+mHncUudr4BRF7/hT0cIFDEhrd8jLhuKYenjzQ4wcqZwNiIr9a7R7Ats+JvjZtLos73qlcsoYytO7asKBVh65dhkDpO2ORMqu5G//2hthdxf6nkdmGsAQFrkq3cfvDW7qlLjc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=cwySrBey; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="cwySrBey" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 41B58C19424; Mon, 3 Nov 2025 07:19:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1762154353; bh=wzU2Qnm2GHt6DLOx5HoozrHwNMWUtEvwOpzlAH8V7/A=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=cwySrBeyrHwqMf9to0lO3sOlJKkPtKOMXAsK6+2zva43tRL6JRWPA3UnfMxLCnmE+ Edx9nSmuQSQER9WZndKlYUGLPDB+K0dgFASq+JHOxPdAitsazq1mGU3689YehQNeIF bqDGfBR15LIjiGtgxU6MJ4UMcPyPiAAF6OZuO3G1mu1o+/NhEzpTas1oxfd1XsLFTS 2ksjoLtHJ/QzN3bT/LKhkVEF4prpJW/XIRH/rZSi6EuZC5BuhW4Z7ShsgF8ixl+fa+ TND3OMU8XUYiIVz86s12iGvQVfP34HqWw2uMKfAEpknNoggqoCy8jv4+5gthttbT5+ f7HgD/XHcVwCA== Received: by paulmck-ThinkPad-P17-Gen-1.home (Postfix, from userid 1000) id E60B4CE0F65; Sun, 2 Nov 2025 14:49:49 -0800 (PST) From: "Paul E. McKenney" To: rcu@vger.kernel.org Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com, rostedt@goodmis.org, "Paul E. McKenney" , Mathieu Desnoyers , Sebastian Andrzej Siewior , bpf@vger.kernel.org Subject: [PATCH 1/8] refscale: Exercise DEFINE_STATIC_SRCU_FAST() and init_srcu_struct_fast() Date: Sun, 2 Nov 2025 14:49:41 -0800 Message-Id: <20251102224948.3906224-1-paulmck@kernel.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> References: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This commit updates the initialization for the "srcu-fast" scale type to use DEFINE_STATIC_SRCU_FAST() when reader_flavor is equal to SRCU_READ_FLAVOR_FAST. Signed-off-by: Paul E. McKenney Cc: Mathieu Desnoyers Cc: Steven Rostedt Cc: Sebastian Andrzej Siewior Cc: --- kernel/rcu/refscale.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 19841704d8f5..ece77f6d055b 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -184,6 +184,7 @@ static const struct ref_scale_ops rcu_ops =3D { =20 // Definitions for SRCU ref scale testing. DEFINE_STATIC_SRCU(srcu_refctl_scale); +DEFINE_STATIC_SRCU_FAST(srcu_fast_refctl_scale); static struct srcu_struct *srcu_ctlp =3D &srcu_refctl_scale; =20 static void srcu_ref_scale_read_section(const int nloops) @@ -216,6 +217,12 @@ static const struct ref_scale_ops srcu_ops =3D { .name =3D "srcu" }; =20 +static bool srcu_fast_sync_scale_init(void) +{ + srcu_ctlp =3D &srcu_fast_refctl_scale; + return true; +} + static void srcu_fast_ref_scale_read_section(const int nloops) { int i; @@ -240,7 +247,7 @@ static void srcu_fast_ref_scale_delay_section(const int= nloops, const int udl, c } =20 static const struct ref_scale_ops srcu_fast_ops =3D { - .init =3D rcu_sync_scale_init, + .init =3D srcu_fast_sync_scale_init, .readsection =3D srcu_fast_ref_scale_read_section, .delaysection =3D srcu_fast_ref_scale_delay_section, .name =3D "srcu-fast" --=20 2.40.1 From nobody Mon Feb 9 04:27:44 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 58D9E2BE7C3; Mon, 3 Nov 2025 07:19:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154354; cv=none; b=BQhYQAjckH7WTfjKe/6r4ElWXx3V45atlKHZKZ94dkPR8S+uRmRuuMhdsw5/el7FFXnu+0QIGg6CceIqL/h19cBv2Kyss0GFbHGjkX3SUoUtsyVsS4QE00p12ZpZUagLGz52iOKsoszx4wdj6Lb2vxZSMI+IcGaeGtb9se+BAhI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154354; c=relaxed/simple; bh=QSNK7smghetxwTK4O0dGzA6u/s8ZwbjtzYihf+jP9hU=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=RAZVavAa6Jp6sGHnSojAxN7c9xewkE8qtqCbc9ZSsnOVosfmm5KuylHO7F9aRFEOsxv7tpup0L1lY0obKNyAE2TaY7CVeNhaWLhvrUCzfevOVYpA+iJFSnTMUiUFelN+GaR+RsM1YSUT/oFbYCh9d/iq9znc4WSG1dY9isleob4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=sb3eJfad; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="sb3eJfad" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 31CCEC19425; Mon, 3 Nov 2025 07:19:14 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1762154354; bh=QSNK7smghetxwTK4O0dGzA6u/s8ZwbjtzYihf+jP9hU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=sb3eJfadfbdZf4W+HdKeBdXRa8rwC0JrRf03zVFFXGf6lwsFZ/ldpCceh1b1ynSrR vkQJH/VMWcFhjh76JhOrPRgAyebVN3qDt7LQzMh0hXN7f53SKVhkYoqEw88cfvU4nC D+4bqlHS/QaptVzXEmxkEEIsEh1EAD2F7brZuEex51LLg0C1pQfMa4BbFCEGOra/FD IAebUtbo0unNpfWpKN5VIN47Do1uTJ5hkjT5t7zdOghOfVw3EGYZw4PFI2ao2+BgmC 22cQe/olaYsGnYSo2ZQ8DwIqezx33tG91EuVV+hXsbtVqf1E8tHZtJDrumR4kcem9f FG7Usq7aDZV6Q== Received: by paulmck-ThinkPad-P17-Gen-1.home (Postfix, from userid 1000) id E8DFFCE0FF8; Sun, 2 Nov 2025 14:49:49 -0800 (PST) From: "Paul E. McKenney" To: rcu@vger.kernel.org Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com, rostedt@goodmis.org, "Paul E. McKenney" Subject: [PATCH 2/8] refscale: Add local_irq_disable() and local_irq_save() readers Date: Sun, 2 Nov 2025 14:49:42 -0800 Message-Id: <20251102224948.3906224-2-paulmck@kernel.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> References: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This commit adds refscale readers based on local_irq_disable() and local_irq_enable() ("refscale.scale_type=3Dirq") and on local_irq_save() and local_irq_restore ("refscale.scale_type=3Dirqsave"). On my x86 laptop, these are about 2.8ns and 7.5ns per enable/disable pair, respectively. Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 66 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index ece77f6d055b..266a3fa94b54 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -636,6 +636,70 @@ static const struct ref_scale_ops jiffies_ops =3D { .name =3D "jiffies" }; =20 +static void ref_irq_section(const int nloops) +{ + int i; + + preempt_disable(); + for (i =3D nloops; i >=3D 0; i--) { + local_irq_disable(); + local_irq_enable(); + } + preempt_enable(); +} + +static void ref_irq_delay_section(const int nloops, const int udl, const i= nt ndl) +{ + int i; + + preempt_disable(); + for (i =3D nloops; i >=3D 0; i--) { + local_irq_disable(); + un_delay(udl, ndl); + local_irq_enable(); + } + preempt_enable(); +} + +static const struct ref_scale_ops irq_ops =3D { + .readsection =3D ref_irq_section, + .delaysection =3D ref_irq_delay_section, + .name =3D "irq" +}; + +static void ref_irqsave_section(const int nloops) +{ + unsigned long flags; + int i; + + preempt_disable(); + for (i =3D nloops; i >=3D 0; i--) { + local_irq_save(flags); + local_irq_restore(flags); + } + preempt_enable(); +} + +static void ref_irqsave_delay_section(const int nloops, const int udl, con= st int ndl) +{ + unsigned long flags; + int i; + + preempt_disable(); + for (i =3D nloops; i >=3D 0; i--) { + local_irq_save(flags); + un_delay(udl, ndl); + local_irq_restore(flags); + } + preempt_enable(); +} + +static const struct ref_scale_ops irqsave_ops =3D { + .readsection =3D ref_irqsave_section, + .delaysection =3D ref_irqsave_delay_section, + .name =3D "irqsave" +}; + //////////////////////////////////////////////////////////////////////// // // Methods leveraging SLAB_TYPESAFE_BY_RCU. @@ -1172,7 +1236,7 @@ ref_scale_init(void) static const struct ref_scale_ops *scale_ops[] =3D { &rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, - &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, + &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, &irq_ops, &irqs= ave_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, }; =20 --=20 2.40.1 From nobody Mon Feb 9 04:27:44 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A16E92BCF54; Mon, 3 Nov 2025 07:19:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154353; cv=none; b=TpYBgFzsLUEgKIdbiGUDJRBbnKdLmpwjMddF53KE+ynwzVXsoK+cfdLJK4myw7wg61Wrvw/MtuWFdHdkHiPYQKPhf6xSSRmFo6AjPDS5l2827YpOGfpvc2RQJaj8MUWpv5aUiBcp3QNF7QFntgERJ2Th9JhNLIc0AvRRAB5Dpfw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154353; c=relaxed/simple; bh=9BQtYTW3cHMDKjYUKt5ajghnwRdt9/BJ0CAqZSRRKGY=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=kOiRRKSpeG/K8ZRwecIc6epnd2ft1C91xjnARGv8W5CaAsbO8/Rq6n7BkpuYPdR81KCZT1fsxJeqO4y7aargbXNSsbRqYSEbp++HJhp7FRpCwBsq+uObjhAtYySLMNHgnWjvZ6YCOmB79dR6I7aKqZaKyT8pjoVvT+6Cnjc3qgc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=f15oXOG0; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="f15oXOG0" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 43F5AC4AF09; Mon, 3 Nov 2025 07:19:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1762154353; bh=9BQtYTW3cHMDKjYUKt5ajghnwRdt9/BJ0CAqZSRRKGY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=f15oXOG0GdvLlDZXnXqDvwKiR7A6FLAhGnxcZeY9f7xArdDA3sGf4w8rlr7/ZVHQV DRhuap1Z/g1aTt+u/IlZcIV8VjDEjW0kI2mT3W+r1LgNepPmhN4ExMc7qUu1AmBMle lu2jJQ/Oss+vXIU0va80yTmR5iA+tPgQOzR8blMp1oGGqejpEV62kbPKdNqgadtIsM vSOY9wWuO6u6fWnewLbCJ5nCZkk4vfwfAb8YGWn+MR3l3qeGVQAuEIiDomFqM6vVHB 4vGDR5+Pz2vq9A8wa72jjiN9IX2R2+dzVKBvN6ZjuRLbOhNfx+SEZMslggLXhxhice NpIQcrWHI6EvA== Received: by paulmck-ThinkPad-P17-Gen-1.home (Postfix, from userid 1000) id EB6F5CE1104; Sun, 2 Nov 2025 14:49:49 -0800 (PST) From: "Paul E. McKenney" To: rcu@vger.kernel.org Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com, rostedt@goodmis.org, "Paul E. McKenney" Subject: [PATCH 3/8] refscale: Add local_bh_disable() readers Date: Sun, 2 Nov 2025 14:49:43 -0800 Message-Id: <20251102224948.3906224-3-paulmck@kernel.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> References: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This commit adds refscale readers based on local_bh_disable() and local_bh_enable() ("refscale.scale_type=3Dbh"). On my x86 laptop, these are about 4.9ns. Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 266a3fa94b54..1faed90231ab 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -636,6 +636,37 @@ static const struct ref_scale_ops jiffies_ops =3D { .name =3D "jiffies" }; =20 +static void ref_bh_section(const int nloops) +{ + int i; + + preempt_disable(); + for (i =3D nloops; i >=3D 0; i--) { + local_bh_disable(); + local_bh_enable(); + } + preempt_enable(); +} + +static void ref_bh_delay_section(const int nloops, const int udl, const in= t ndl) +{ + int i; + + preempt_disable(); + for (i =3D nloops; i >=3D 0; i--) { + local_bh_disable(); + un_delay(udl, ndl); + local_bh_enable(); + } + preempt_enable(); +} + +static const struct ref_scale_ops bh_ops =3D { + .readsection =3D ref_bh_section, + .delaysection =3D ref_bh_delay_section, + .name =3D "bh" +}; + static void ref_irq_section(const int nloops) { int i; @@ -1236,7 +1267,8 @@ ref_scale_init(void) static const struct ref_scale_ops *scale_ops[] =3D { &rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, - &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, &irq_ops, &irqs= ave_ops, + &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, + &bh_ops, &irq_ops, &irqsave_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, }; =20 --=20 2.40.1 From nobody Mon Feb 9 04:27:44 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7F20B2BEC4A; Mon, 3 Nov 2025 07:19:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154354; cv=none; b=J/CQ2lEk2UOTKitxXOjrPqiBriyf5cXAay/vjncK2SbrmQKFiwd9FdMW6k8MZR6IyVicZ64K6219l3lf7e4t6FYCDB4QaaqYyp5AGfcRsGTOoSpPlbCN0K1LrCD6u2bv/wv/Jx6oh06DVZjdBjyxJfQnz8cPq0Ekm4rFPHbDoxI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154354; c=relaxed/simple; bh=LuhGHYxKKiGIAFnwvtKfy9cMeJ9OKzRJZ6z1CGVEEN4=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=dTjV3ZFDzTTZaFfFpHdkFb0wV/XJ0+cY28JMrcqakBWsLI6lxhr9XcvZYsZj6q013gdUFZeFByxAcuzQYnnaFNU7ifeNxZWZH8Z2oWgh6MDLOWCnGbayfXK1UxREZrDQUDFmSLR1LBxqZ4VLNGXU7z+8JOD/HCMp2B03Yc3m5VA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=aoXkWsu0; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="aoXkWsu0" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 47502C116D0; Mon, 3 Nov 2025 07:19:14 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1762154354; bh=LuhGHYxKKiGIAFnwvtKfy9cMeJ9OKzRJZ6z1CGVEEN4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=aoXkWsu0Jv+mg1UfXUsDOiZUs81R4R+rk1KA4mpA+MQ5OwBF62BAR0H6e/VGGbKMb o+d1pgunHZ9BkSeOuNHSWREPMiKIGoKvkZ3RUQSh24Cs474dIfbTKqd+Al1Yvelwjr Fo5oeOSF7G2guDdv8zrSAAAVPfvUB3/f4em2aQ4whp42h8fZyCBB9yPi/CjTNvX6KH MHCb78wsxOq8csnWfThl9m+U7G9hz3tkzhPHR7kb/c4+P5pYQybi0or7pxoVftsCHy nB1adMokSWPB0HtOMl5jB3NsYSX1XtPO3FxEi72USUUyZFwgXEw6iETy0ESCecLQz2 PYtAf2G75SR4Q== Received: by paulmck-ThinkPad-P17-Gen-1.home (Postfix, from userid 1000) id EE471CE122D; Sun, 2 Nov 2025 14:49:49 -0800 (PST) From: "Paul E. McKenney" To: rcu@vger.kernel.org Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com, rostedt@goodmis.org, "Paul E. McKenney" Subject: [PATCH 4/8] refscale: Add preempt_disable() readers Date: Sun, 2 Nov 2025 14:49:44 -0800 Message-Id: <20251102224948.3906224-4-paulmck@kernel.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> References: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This commit adds refscale readers based on preempt_disable() and preempt_enable() ("refscale.scale_type=3Dpreempt"). On my x86 laptop, these are about 2.8ns. Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 1faed90231ab..ac9a0b6332ac 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -636,6 +636,37 @@ static const struct ref_scale_ops jiffies_ops =3D { .name =3D "jiffies" }; =20 +static void ref_preempt_section(const int nloops) +{ + int i; + + migrate_disable(); + for (i =3D nloops; i >=3D 0; i--) { + preempt_disable(); + preempt_enable(); + } + migrate_enable(); +} + +static void ref_preempt_delay_section(const int nloops, const int udl, con= st int ndl) +{ + int i; + + migrate_disable(); + for (i =3D nloops; i >=3D 0; i--) { + preempt_disable(); + un_delay(udl, ndl); + preempt_enable(); + } + migrate_enable(); +} + +static const struct ref_scale_ops preempt_ops =3D { + .readsection =3D ref_preempt_section, + .delaysection =3D ref_preempt_delay_section, + .name =3D "preempt" +}; + static void ref_bh_section(const int nloops) { int i; @@ -1268,7 +1299,7 @@ ref_scale_init(void) &rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, - &bh_ops, &irq_ops, &irqsave_ops, + &preempt_ops, &bh_ops, &irq_ops, &irqsave_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, }; =20 --=20 2.40.1 From nobody Mon Feb 9 04:27:44 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B80202BD5A2; Mon, 3 Nov 2025 07:19:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154353; cv=none; b=pV/kEGu4EHgdJQXbyxm9SJmYSV+dv8//Z/dA22C9YnUmaOFZaqHncT2vgsyPmXYX35Et7KdDVipOE5gLuyZdx8vt2jn63FSiCgm08LIuNutvxIRDD1mQuJORdNOrTwrA9cDnZ0L56swYeUOkXcQcByyxuV8QqoTILHUiYMEmA0g= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154353; c=relaxed/simple; bh=1F2Dota48fNMOMG+prFXxULTEjMwm4pw9eYEAGb2MqI=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=tcTCmZY7LPuzI2uI+rdoHRD3b8NoyQS/FURCWso3S2FpsMWut7g4qDDrAj2uAselUQe9U4vsID5B7bsV5aT3PSKWORyT2UxEHVcxfKJYtf+9FKx106l/igEOr1czSabHxMMeXStet2F+qjY2vfErh1yC6CnmUEBpPrcf3LYM73g= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=InCRKSvf; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="InCRKSvf" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 65639C2BCB6; Mon, 3 Nov 2025 07:19:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1762154353; bh=1F2Dota48fNMOMG+prFXxULTEjMwm4pw9eYEAGb2MqI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=InCRKSvfYpOox/bY9s1vopDyRlCruUv1OsPtKMkZznmc2icLdh3zzpx1849eO7nRd nADpa0iPIwGho06ZIs/koMcoRr7bShwdRiSEDRnpUQtiOfdF7c50JjDTpM2741NKgd 8HBEcla+3p/coyawbAp2GcKijUR/fjtLm78rotPP3YyUFRrlx6naqbGEVIQ3lQEiOw QaD1Uj+WGIhStoVE6kjMfXYRHyRQGEkpCPCiP4s+JkdnptLtvJCkiyByzXnKng9+qZ OEK3pZ20UshAQfgPI8ZlgQPv0ahdJR9ThZ0RKjs3dn2FKl8TGnOKi3o07seH88G8vd hV6lM+d2w7J/w== Received: by paulmck-ThinkPad-P17-Gen-1.home (Postfix, from userid 1000) id F0B70CE1257; Sun, 2 Nov 2025 14:49:49 -0800 (PST) From: "Paul E. McKenney" To: rcu@vger.kernel.org Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com, rostedt@goodmis.org, "Paul E. McKenney" Subject: [PATCH 5/8] refscale: Add this_cpu_inc() readers Date: Sun, 2 Nov 2025 14:49:45 -0800 Message-Id: <20251102224948.3906224-5-paulmck@kernel.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> References: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This commit adds refscale readers based on this_cpu_inc() and this_cpu_inc() ("refscale.scale_type=3Dpercpuinc"). On my x86 laptop, these are about 4.5ns per pair. Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index ac9a0b6332ac..479d86cad652 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -330,6 +330,9 @@ static const struct ref_scale_ops rcu_trace_ops =3D { // Definitions for reference count static atomic_t refcnt; =20 +// Definitions acquire-release. +static DEFINE_PER_CPU(unsigned long, test_acqrel); + static void ref_refcnt_section(const int nloops) { int i; @@ -358,6 +361,34 @@ static const struct ref_scale_ops refcnt_ops =3D { .name =3D "refcnt" }; =20 +static void ref_percpuinc_section(const int nloops) +{ + int i; + + for (i =3D nloops; i >=3D 0; i--) { + this_cpu_inc(test_acqrel); + this_cpu_dec(test_acqrel); + } +} + +static void ref_percpuinc_delay_section(const int nloops, const int udl, c= onst int ndl) +{ + int i; + + for (i =3D nloops; i >=3D 0; i--) { + this_cpu_inc(test_acqrel); + un_delay(udl, ndl); + this_cpu_dec(test_acqrel); + } +} + +static const struct ref_scale_ops percpuinc_ops =3D { + .init =3D rcu_sync_scale_init, + .readsection =3D ref_percpuinc_section, + .delaysection =3D ref_percpuinc_delay_section, + .name =3D "percpuinc" +}; + // Definitions for rwlock static rwlock_t test_rwlock; =20 @@ -501,9 +532,6 @@ static const struct ref_scale_ops lock_irq_ops =3D { .name =3D "lock-irq" }; =20 -// Definitions acquire-release. -static DEFINE_PER_CPU(unsigned long, test_acqrel); - static void ref_acqrel_section(const int nloops) { unsigned long x; @@ -1298,7 +1326,7 @@ ref_scale_init(void) static const struct ref_scale_ops *scale_ops[] =3D { &rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, - &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, + &percpuinc_ops, &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, &preempt_ops, &bh_ops, &irq_ops, &irqsave_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, }; --=20 2.40.1 From nobody Mon Feb 9 04:27:44 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8BCAB29E113; Mon, 3 Nov 2025 07:19:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154353; cv=none; b=q/Ug7UKUK+bsHMr+MyaFKGSPgvakJ5UrZ2QPPmoGih0Kaep+P2dJ4S2re/cEjaH8SK/BU3dsHABi1BHLpYGPgia2rRCMoC+81hb04maKRukBBdklidWaI9lJrJkSLHOgdT8UlWk/ireMOoUfSu9YMcVSMMA+Xrbp4Qtp/dMDsBs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154353; c=relaxed/simple; bh=rAn+D1aKec9450avC0ni1vP7cPV1yyUFx1owntShnrM=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=NNn+mBNOwu/ejc2nQ+mgqzb88Xnl+J7l3xo2bHhfjSv7znNowJgHngiZYiuyHaModkbSBoopIYT9bXB+V3+Z9g3LPK1rsPWBfc7I/GxLXTs1icK/Q335nKj2BR4mj2Mcl5Ce0r7CIhKQBt8G6HQHcOIsqMuW3G8qD3xs3ENS5Oo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=XCWh5bCo; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="XCWh5bCo" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 35C6BC19423; Mon, 3 Nov 2025 07:19:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1762154353; bh=rAn+D1aKec9450avC0ni1vP7cPV1yyUFx1owntShnrM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=XCWh5bCooV/BYDOiUgheIY7JAnyKGdfFKWJCw+z5Jz2LxF41Duczb2nhMfaHBjFX5 R33EGyxiiY6RRSYF02QTwnAvXuJF606g3wxJR05l5yO3zjiZVzTNqm/ATCEzuCFXz2 YF2d7vjgAAaHbWIXtL3j0wilbmwtG0RM0mAgOpMUAPxdgzn4fJqfIfZSbV4kNq+AnU jJYfLoGNzQGPPnisai7w/rIe8uPbrfgTNFV+hrPEoYwaO+4cX5uM2oNGVoYeTpJBCg yQSnrhTCXeKV3T42sslsIyRIQTc3Q7YFvTRCCiqx3Abe12LyjRCkXd/Ufrb6PrGlgP 0gLDHv0HCzaUg== Received: by paulmck-ThinkPad-P17-Gen-1.home (Postfix, from userid 1000) id F33F4CE12A7; Sun, 2 Nov 2025 14:49:49 -0800 (PST) From: "Paul E. McKenney" To: rcu@vger.kernel.org Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com, rostedt@goodmis.org, "Paul E. McKenney" Subject: [PATCH 6/8] refscale: Add non-atomic per-CPU increment readers Date: Sun, 2 Nov 2025 14:49:46 -0800 Message-Id: <20251102224948.3906224-6-paulmck@kernel.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> References: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This commit adds refscale readers based on READ_ONCE() and WRITE_ONCE() that are unprotected (can lose counts, "refscale.scale_type=3Dincpercpu"), preempt-disabled ("refscale.scale_type=3Dincpercpupreempt"), bh-disabled ("refscale.scale_type=3Dincpercpubh"), and irq-disabled ("refscale.scale_type=3Dincpercpuirqsave"). On my x86 laptop, these are about 4.3ns, 3.8ns, and 7.3ns per pair, respectively. Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 155 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 479d86cad652..2b247355de40 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -389,6 +389,155 @@ static const struct ref_scale_ops percpuinc_ops =3D { .name =3D "percpuinc" }; =20 +// Note that this can lose counts in preemptible kernels. +static void ref_incpercpu_section(const int nloops) +{ + int i; + + for (i =3D nloops; i >=3D 0; i--) { + unsigned long *tap =3D this_cpu_ptr(&test_acqrel); + + WRITE_ONCE(*tap, READ_ONCE(*tap) + 1); + WRITE_ONCE(*tap, READ_ONCE(*tap) - 1); + } +} + +static void ref_incpercpu_delay_section(const int nloops, const int udl, c= onst int ndl) +{ + int i; + + for (i =3D nloops; i >=3D 0; i--) { + unsigned long *tap =3D this_cpu_ptr(&test_acqrel); + + WRITE_ONCE(*tap, READ_ONCE(*tap) + 1); + un_delay(udl, ndl); + WRITE_ONCE(*tap, READ_ONCE(*tap) - 1); + } +} + +static const struct ref_scale_ops incpercpu_ops =3D { + .init =3D rcu_sync_scale_init, + .readsection =3D ref_incpercpu_section, + .delaysection =3D ref_incpercpu_delay_section, + .name =3D "incpercpu" +}; + +static void ref_incpercpupreempt_section(const int nloops) +{ + int i; + + for (i =3D nloops; i >=3D 0; i--) { + unsigned long *tap; + + preempt_disable(); + tap =3D this_cpu_ptr(&test_acqrel); + WRITE_ONCE(*tap, READ_ONCE(*tap) + 1); + WRITE_ONCE(*tap, READ_ONCE(*tap) - 1); + preempt_enable(); + } +} + +static void ref_incpercpupreempt_delay_section(const int nloops, const int= udl, const int ndl) +{ + int i; + + for (i =3D nloops; i >=3D 0; i--) { + unsigned long *tap; + + preempt_disable(); + tap =3D this_cpu_ptr(&test_acqrel); + WRITE_ONCE(*tap, READ_ONCE(*tap) + 1); + un_delay(udl, ndl); + WRITE_ONCE(*tap, READ_ONCE(*tap) - 1); + preempt_enable(); + } +} + +static const struct ref_scale_ops incpercpupreempt_ops =3D { + .init =3D rcu_sync_scale_init, + .readsection =3D ref_incpercpupreempt_section, + .delaysection =3D ref_incpercpupreempt_delay_section, + .name =3D "incpercpupreempt" +}; + +static void ref_incpercpubh_section(const int nloops) +{ + int i; + + for (i =3D nloops; i >=3D 0; i--) { + unsigned long *tap; + + local_bh_disable(); + tap =3D this_cpu_ptr(&test_acqrel); + WRITE_ONCE(*tap, READ_ONCE(*tap) + 1); + WRITE_ONCE(*tap, READ_ONCE(*tap) - 1); + local_bh_enable(); + } +} + +static void ref_incpercpubh_delay_section(const int nloops, const int udl,= const int ndl) +{ + int i; + + for (i =3D nloops; i >=3D 0; i--) { + unsigned long *tap; + + local_bh_disable(); + tap =3D this_cpu_ptr(&test_acqrel); + WRITE_ONCE(*tap, READ_ONCE(*tap) + 1); + un_delay(udl, ndl); + WRITE_ONCE(*tap, READ_ONCE(*tap) - 1); + local_bh_enable(); + } +} + +static const struct ref_scale_ops incpercpubh_ops =3D { + .init =3D rcu_sync_scale_init, + .readsection =3D ref_incpercpubh_section, + .delaysection =3D ref_incpercpubh_delay_section, + .name =3D "incpercpubh" +}; + +static void ref_incpercpuirqsave_section(const int nloops) +{ + int i; + unsigned long flags; + + for (i =3D nloops; i >=3D 0; i--) { + unsigned long *tap; + + local_irq_save(flags); + tap =3D this_cpu_ptr(&test_acqrel); + WRITE_ONCE(*tap, READ_ONCE(*tap) + 1); + WRITE_ONCE(*tap, READ_ONCE(*tap) - 1); + local_irq_restore(flags); + } +} + +static void ref_incpercpuirqsave_delay_section(const int nloops, const int= udl, const int ndl) +{ + int i; + unsigned long flags; + + for (i =3D nloops; i >=3D 0; i--) { + unsigned long *tap; + + local_irq_save(flags); + tap =3D this_cpu_ptr(&test_acqrel); + WRITE_ONCE(*tap, READ_ONCE(*tap) + 1); + un_delay(udl, ndl); + WRITE_ONCE(*tap, READ_ONCE(*tap) - 1); + local_irq_restore(flags); + } +} + +static const struct ref_scale_ops incpercpuirqsave_ops =3D { + .init =3D rcu_sync_scale_init, + .readsection =3D ref_incpercpuirqsave_section, + .delaysection =3D ref_incpercpuirqsave_delay_section, + .name =3D "incpercpuirqsave" +}; + // Definitions for rwlock static rwlock_t test_rwlock; =20 @@ -1325,8 +1474,10 @@ ref_scale_init(void) int firsterr =3D 0; static const struct ref_scale_ops *scale_ops[] =3D { &rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS - &refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, - &percpuinc_ops, &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, + &refcnt_ops, &percpuinc_ops, &incpercpu_ops, &incpercpupreempt_ops, + &incpercpubh_ops, &incpercpuirqsave_ops, + &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, + &sched_clock_ops, &clock_ops, &jiffies_ops, &preempt_ops, &bh_ops, &irq_ops, &irqsave_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, }; --=20 2.40.1 From nobody Mon Feb 9 04:27:44 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 933BA298CC7; Mon, 3 Nov 2025 07:19:12 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154352; cv=none; b=j6Rgryl1+JifpIPqRwxeu/e5D1STbCa1u76mS4/LxeW0IyVrbWYNpEiZFk39JPOOrpaD+XZZBGY36jpAQA8xc3txp3mkc/ywlpJWnY6Nb7zTdonUN9o8IWDqeqZQyKqDclElJyTRKSjGSOk1VvRxYiXwFI9wGhQFsAN0VKl+khI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154352; c=relaxed/simple; bh=d+Yn3i1lzgRjb0Q5CeMOtXQwj+BmYuzfNnrAziwjsOA=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=YeFu8O+4AXbBNkShijA4zXlyeyVV/oPuUFKlp3lZpt4bbKvEmbWRQZzk4pphp1drHNH7oIC/zTcmMBpKoc3MToUwXJrk4dkZmBtN3f8a9dwHmOVkAl6y/Uney9gCvPzK/iLKIrpFJsDUb4vvQaRN2CMzfyUCZKEx64zOL7cKcFg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=gBa2rhcF; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="gBa2rhcF" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 36172C4CEFD; Mon, 3 Nov 2025 07:19:12 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1762154352; bh=d+Yn3i1lzgRjb0Q5CeMOtXQwj+BmYuzfNnrAziwjsOA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=gBa2rhcFvKwFNpcXBJrF+Z0Q2utbsvC+ckcki/Fk/XaUl816LJjpLORp38k11FxS4 6cIX8pVEDaEKtrdmEjdRqnipQKoEOkmm6WSsWGHpMJDZyzSqIMZAZif2R3dchUtsYk vVvyfmGCygm+wcqOq2Pgz5utpUHvatRme9iuSfrzl6GNHe6p1VulqWelhItiMscTZ8 FZfuIitkKXA5uPexsT2eVJ32T6w7tTxSfk+0IW7aXXzZuPSQD2vt3g4z/Q1IJQFOmT o2hhInf6pvZc5UeSzICOavGUMVqtZhbT0b8Ij+EyHkG4hd8Eo/NuJlsN0E22E5fLW8 f/PYNWZbGb5BA== Received: by paulmck-ThinkPad-P17-Gen-1.home (Postfix, from userid 1000) id 019E0CE13CA; Sun, 2 Nov 2025 14:49:50 -0800 (PST) From: "Paul E. McKenney" To: rcu@vger.kernel.org Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com, rostedt@goodmis.org, "Paul E. McKenney" Subject: [PATCH 7/8] refscale: Do not diable interrupts for tests involving local_bh_enable() Date: Sun, 2 Nov 2025 14:49:47 -0800 Message-Id: <20251102224948.3906224-7-paulmck@kernel.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> References: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Some kernel configurations prohibit invoking local_bh_enable() while interrupts are disabled. However, refscale disables interrupts to reduce OS noise during the tests, which results in splats. This commit therefore adds an ->enable_irqs flag to the ref_scale_ops structure, and refrains from disabling interrupts when that flag is set. This flag is set for the "bh" and "incpercpubh" scale_type module-parameter values. Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 2b247355de40..7429ec9f0092 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -136,6 +136,7 @@ struct ref_scale_ops { void (*cleanup)(void); void (*readsection)(const int nloops); void (*delaysection)(const int nloops, const int udl, const int ndl); + bool enable_irqs; const char *name; }; =20 @@ -495,6 +496,7 @@ static const struct ref_scale_ops incpercpubh_ops =3D { .init =3D rcu_sync_scale_init, .readsection =3D ref_incpercpubh_section, .delaysection =3D ref_incpercpubh_delay_section, + .enable_irqs =3D true, .name =3D "incpercpubh" }; =20 @@ -872,6 +874,7 @@ static void ref_bh_delay_section(const int nloops, cons= t int udl, const int ndl) static const struct ref_scale_ops bh_ops =3D { .readsection =3D ref_bh_section, .delaysection =3D ref_bh_delay_section, + .enable_irqs =3D true, .name =3D "bh" }; =20 @@ -1234,15 +1237,18 @@ ref_scale_reader(void *arg) if (!atomic_dec_return(&n_warmedup)) while (atomic_read_acquire(&n_warmedup)) rcu_scale_one_reader(); - // Also keep interrupts disabled. This also has the effect - // of preventing entries into slow path for rcu_read_unlock(). - local_irq_save(flags); + // Also keep interrupts disabled when it is safe to do so, which + // it is not for local_bh_enable(). This also has the effect of + // preventing entries into slow path for rcu_read_unlock(). + if (!cur_ops->enable_irqs) + local_irq_save(flags); start =3D ktime_get_mono_fast_ns(); =20 rcu_scale_one_reader(); =20 duration =3D ktime_get_mono_fast_ns() - start; - local_irq_restore(flags); + if (!cur_ops->enable_irqs) + local_irq_restore(flags); =20 rt->last_duration_ns =3D WARN_ON_ONCE(duration < 0) ? 0 : duration; // To reduce runtime-skew noise, do maintain-load invocations until --=20 2.40.1 From nobody Mon Feb 9 04:27:44 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9EFE72BEFF9; Mon, 3 Nov 2025 07:19:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154354; cv=none; b=tqtGZI9/Y+17C9RWZqbyOlfgnuRSRL8iNkaEqLqN9S4aS8efEnyWMR0zGNCn5kD5XU9f4LMZru/azRF97D4bmc01OTxmUiWyfroa5Gm/LCLsFinTMezcTBt5He02gy636OdrxCdxlSqJqm3e80ZbAONKAUFqBxRP0TJRN7uapVU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1762154354; c=relaxed/simple; bh=8ZCRNXXqmbiShzfe05dmFtCIZXEOypE5e1dE70Rypfo=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=SrmVUqewK4ae0LS7nEjq073MUdqDYkpOxWnzeNQFs/PECsUQTd0dqllCMB0uCJ3aRNvqdUYm/dsZVhGlFM1iKkvIG5ERC3jXEaJgFecwR+sNmfu8aTfcBUKMHFFDyYKuheTs+buL6FnCRs2fMKAPUGR62a4+72yp6MWvnYeN10U= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=uWSg0cJ3; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="uWSg0cJ3" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 78A66C19423; Mon, 3 Nov 2025 07:19:14 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1762154354; bh=8ZCRNXXqmbiShzfe05dmFtCIZXEOypE5e1dE70Rypfo=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=uWSg0cJ31rkYVJyjqhuAnue/g4l4rIK0s8Konh5jLRD36uxNAaFltUbqKR/ocyvNj u8UV+2QAWAWSbnKTrQrkCyu4LD+3DPPVdnsVB9CToWPEJyg6MOcvieckyqoS88BgMy 1nWfYssSxzanfQRhtW4G/QSIDtPlfbn1W3tToDx0yrZ6//SJuYMTICHXoCu4plIemz pH7d/8Ls52k3Cw7gVJl1sa0AaJgwO18qXv0RDuW/lYMsHFHRQWtkn13n9qAoOeVRWx iSnouZb7vALblJe7Txv0uGeCtBuFE4TvReUz1SKKu1HolxAAOdU82xSS5g9YXb0//y Qw3HJjINaeTqQ== Received: by paulmck-ThinkPad-P17-Gen-1.home (Postfix, from userid 1000) id 04793CE160B; Sun, 2 Nov 2025 14:49:50 -0800 (PST) From: "Paul E. McKenney" To: rcu@vger.kernel.org Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com, rostedt@goodmis.org, "Paul E. McKenney" , Mathieu Desnoyers , Sebastian Andrzej Siewior , bpf@vger.kernel.org Subject: [PATCH 8/8] refscale: Add SRCU-fast-updown readers Date: Sun, 2 Nov 2025 14:49:48 -0800 Message-Id: <20251102224948.3906224-8-paulmck@kernel.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> References: <19fae851-0c49-43d2-9bbf-913424641ff4@paulmck-laptop> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This commit adds refscale readers based on srcu_read_lock_fast_updown() and srcu_read_lock_fast_updown() ("refscale.scale_type=3Dsrcu-fast-updown"). On my x86 laptop, these are about 2.2ns per pair. Signed-off-by: Paul E. McKenney Cc: Mathieu Desnoyers Cc: Steven Rostedt Cc: Sebastian Andrzej Siewior Cc: --- kernel/rcu/refscale.c | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 7429ec9f0092..07a313782dfd 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -186,6 +186,7 @@ static const struct ref_scale_ops rcu_ops =3D { // Definitions for SRCU ref scale testing. DEFINE_STATIC_SRCU(srcu_refctl_scale); DEFINE_STATIC_SRCU_FAST(srcu_fast_refctl_scale); +DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_fast_updown_refctl_scale); static struct srcu_struct *srcu_ctlp =3D &srcu_refctl_scale; =20 static void srcu_ref_scale_read_section(const int nloops) @@ -254,6 +255,42 @@ static const struct ref_scale_ops srcu_fast_ops =3D { .name =3D "srcu-fast" }; =20 +static bool srcu_fast_updown_sync_scale_init(void) +{ + srcu_ctlp =3D &srcu_fast_updown_refctl_scale; + return true; +} + +static void srcu_fast_updown_ref_scale_read_section(const int nloops) +{ + int i; + struct srcu_ctr __percpu *scp; + + for (i =3D nloops; i >=3D 0; i--) { + scp =3D srcu_read_lock_fast_updown(srcu_ctlp); + srcu_read_unlock_fast_updown(srcu_ctlp, scp); + } +} + +static void srcu_fast_updown_ref_scale_delay_section(const int nloops, con= st int udl, const int ndl) +{ + int i; + struct srcu_ctr __percpu *scp; + + for (i =3D nloops; i >=3D 0; i--) { + scp =3D srcu_read_lock_fast_updown(srcu_ctlp); + un_delay(udl, ndl); + srcu_read_unlock_fast_updown(srcu_ctlp, scp); + } +} + +static const struct ref_scale_ops srcu_fast_updown_ops =3D { + .init =3D srcu_fast_updown_sync_scale_init, + .readsection =3D srcu_fast_updown_ref_scale_read_section, + .delaysection =3D srcu_fast_updown_ref_scale_delay_section, + .name =3D "srcu-fast-updown" +}; + #ifdef CONFIG_TASKS_RCU =20 // Definitions for RCU Tasks ref scale testing: Empty read markers. @@ -1479,7 +1516,8 @@ ref_scale_init(void) long i; int firsterr =3D 0; static const struct ref_scale_ops *scale_ops[] =3D { - &rcu_ops, &srcu_ops, &srcu_fast_ops, RCU_TRACE_OPS RCU_TASKS_OPS + &rcu_ops, &srcu_ops, &srcu_fast_ops, &srcu_fast_updown_ops, + RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &percpuinc_ops, &incpercpu_ops, &incpercpupreempt_ops, &incpercpubh_ops, &incpercpuirqsave_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, --=20 2.40.1