From nobody Tue Oct 7 06:50:55 2025 Received: from mail-pf1-f177.google.com (mail-pf1-f177.google.com [209.85.210.177]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A94D91F582F; Sat, 12 Jul 2025 21:32:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.210.177 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355928; cv=none; b=Troiv5WmMV0W8OXZ1KB1A6QczT5686vz5oStpV72juSb1eoIlyFCT3GcApykZS660hSy+nmg2ckctxMMpl/VdgIuB2/pY9ierBb728zH2tzHZDr7b58IivGb0yHPlPC6d4zptQxzboFmsO4oNhGa4Mt2E3MNjcAv2fsahuvJlv8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355928; c=relaxed/simple; bh=/wdff0O+TBccIF+YWwxyFnAcyvOePCDxb+KptptoB68=; h=From:Date:Subject:MIME-Version:Content-Type:Message-Id:References: In-Reply-To:To:Cc; b=FI2NSIydaOGz6epNV6Fr+c5UQC8SUE5tlRXJjNfGpEr3XDwHbJP+Z1t0s0gYgP2PBhUuk6hkS9U0RrnTLy1yhrNVR60ES0No9TnceMa7xrkpODGCT0+25J5wHadTR6B2rrfOv/iwxzoK5HakQw3jTNK958x2jT2Unl7p5oICKjk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=FIU5p05M; arc=none smtp.client-ip=209.85.210.177 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="FIU5p05M" Received: by mail-pf1-f177.google.com with SMTP id d2e1a72fcca58-747c2cc3419so2796890b3a.2; Sat, 12 Jul 2025 14:32:06 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1752355926; x=1752960726; darn=vger.kernel.org; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:from:to:cc:subject:date:message-id :reply-to; bh=CTtoMD2jUnmzV6fdblunt8Pya7UDWPcBKIFiwvtZyXU=; b=FIU5p05MWYNEzGuiczheuK2Fua0X4Vf/LuLKyT5yzurMPfEDLynPP5y40kPa/QTxa+ Y2LYLpMQKGBMclbIWRQH5Zvow9HSt7N9zM8RoaXrMLTZI5ahbk7gL37J//Zz2QEQXb2e R+YwX9c7/BRC6fARiSX72SisOuv6Xkf1Cs3zRZpShRX4AtyKJ1ATA/nyxs1gEKyUIdeJ +w5lDK1QEKqgIUWyoiwt/wVitOWdga3+41Ql7QqfUKnlrJvQ4vpaka3f0pFpkdgZYu0u YfO8toymYyVo+4fmWvDUXE51ZrAssmXqAWiXzy/T6+4cNuOjJk0ztp/SXCazAYl3sQoe jQQw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752355926; x=1752960726; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=CTtoMD2jUnmzV6fdblunt8Pya7UDWPcBKIFiwvtZyXU=; b=a4jUhgUgXTnj6ynmqPweeYOTs4cDs1353UxURO0k0WVMFei1gO7qQLBTHn36RIyjMh 66qVqujdvpHYRJlkCobcOn/yoH9p1Cd0YOTG19n9AFm+Y2sjgJdE/ALpJEtPm74aH4wF giJlmm3ye7kMWBVA6WwaLhWtdN5Yfhq/xMbMsXeHyFyzaWnUxwLOX5AtnTuFCCXYy5dr q1as9g8yhrDUnDd5gWx4kZCSORYEEw7whqn+kjAPDxr1054oVtjOB7DHx4yBnxEOwKUk GOqAuudDUmkSX+S6nVK8SVDF4UFewfcwMHt0QaSmVxaO9PpHrRBxwLZgrkMRDEoOBBpG AWSw== X-Forwarded-Encrypted: i=1; AJvYcCVz/1U8KiPCmDJhYa0bOQvUt05Lw9D9t8wN2TUIlyxL4PMaKu0zMGRK1zWX+mROxj1TbwseKFXXR6fwHqD7qA==@vger.kernel.org X-Gm-Message-State: AOJu0YwFMI0MHdj5A6y4aWGit72eQkU5So568opHtVf7cnttgkJppkcF Vn1GwLaCRfY/5qyjODMG8FfqHHY0Leiih5LskTlGc5jsk8XLy8qN+dh6 X-Gm-Gg: ASbGnct26ZPrtCMpX4lSY9Dms5W1SN9h/O8YqkFV5bx5JghEX192RTYQhx45mqaOBSs cIr4jGHj7aIqqm0kTi70wQ2088SXneXznPAjo5yjEoYIDMAU7Ub+cDS6YDOIAzt7ZoIGJv32fTP HfgEvCyIWmB/GtrujH347UNtRRzYlZ0W+R55oJsPqDFp/CaH0/yL7R40ng0WvGquWo3m+XUh32F Ay40rhmnZ/kuNn7FiYuFHOG4sgadBh7ng1QKiaja0ttJjBKj8mMXGlJFKzXkgS3j1vKauKAh3/r BFHqniAejdDMnMJSHUdJtrf7Zpkdv+ec6BgzpRBxFAuEwiqHj+22CIakh37aQ3PdInT9f0bENW7 MD3q9xC/K0FPlOgm7ZX/ri40gXcKr1RH/C7WppMCYWHfNNvzCPhgjxWiG X-Google-Smtp-Source: AGHT+IHm5qpR1gjPtQZ/rA3SjPJLCrptEAwSAPqdZIC/RIhV1B17wk9gihcdL3O+6GYe4cBg0KSDew== X-Received: by 2002:a17:90a:e18e:b0:312:1ae9:1529 with SMTP id 98e67ed59e1d1-31c4cd8b792mr13311488a91.27.1752355925629; Sat, 12 Jul 2025 14:32:05 -0700 (PDT) Received: from mitchelllevy.localdomain ([174.127.224.194]) by smtp.gmail.com with ESMTPSA id 98e67ed59e1d1-31c3eb65aa2sm8003935a91.40.2025.07.12.14.32.04 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 12 Jul 2025 14:32:05 -0700 (PDT) From: Mitchell Levy Date: Sat, 12 Jul 2025 14:31:12 -0700 Subject: [PATCH v2 1/5] rust: percpu: introduce a rust API for per-CPU variables Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Message-Id: <20250712-rust-percpu-v2-1-826f2567521b@gmail.com> References: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> In-Reply-To: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> To: Miguel Ojeda , Alex Gaynor , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Andreas Hindborg , Alice Ryhl , Trevor Gross , Andrew Morton , Dennis Zhou , Tejun Heo , Christoph Lameter , Danilo Krummrich , Benno Lossin Cc: linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-mm@kvack.org, Mitchell Levy X-Mailer: b4 0.14.0 X-Developer-Signature: v=1; a=ed25519-sha256; t=1752355923; l=15933; i=levymitchell0@gmail.com; s=20240719; h=from:subject:message-id; bh=/wdff0O+TBccIF+YWwxyFnAcyvOePCDxb+KptptoB68=; b=fGprdJqyWMecWZn0BZns5u9xCisKn6z932FGaZjQybSNKSx6RtIY/nAITGYWL2SRjzCB7MQIi XRcE08T71eXAgh4d8Zxj22bRAI+gzPsvVUhbxuCfGiO09XNkEGjbYkw X-Developer-Key: i=levymitchell0@gmail.com; a=ed25519; pk=n6kBmUnb+UNmjVkTnDwrLwTJAEKUfs2e8E+MFPZI93E= Add a `CpuGuard` type that disables preemption for its lifetime. Add a `PerCpuAllocation` type used to track dynamic allocations. Add a `define_per_cpu!` macro to create static per-CPU allocations. Add `DynamicPerCpu` and `StaticPerCpu` to provide a high-level API. Add a `PerCpu` trait to unify the dynamic and static cases. Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Mitchell Levy --- rust/helpers/helpers.c | 2 + rust/helpers/percpu.c | 9 ++ rust/helpers/preempt.c | 14 ++ rust/kernel/lib.rs | 3 + rust/kernel/percpu.rs | 308 ++++++++++++++++++++++++++++++++++++= ++++ rust/kernel/percpu/cpu_guard.rs | 35 +++++ 6 files changed, 371 insertions(+) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0f1b5d115985..d56bbe6334d3 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -29,7 +29,9 @@ #include "page.c" #include "platform.c" #include "pci.c" +#include "percpu.c" #include "pid_namespace.c" +#include "preempt.c" #include "rbtree.c" #include "rcu.c" #include "refcount.c" diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c new file mode 100644 index 000000000000..a091389f730f --- /dev/null +++ b/rust/helpers/percpu.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) +{ + return __alloc_percpu(sz, align); +} + diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c new file mode 100644 index 000000000000..2c7529528ddd --- /dev/null +++ b/rust/helpers/preempt.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_preempt_disable(void) +{ + preempt_disable(); +} + +void rust_helper_preempt_enable(void) +{ + preempt_enable(); +} + diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 6b4774b2b1c3..733f9ff8b888 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -95,6 +95,9 @@ pub mod page; #[cfg(CONFIG_PCI)] pub mod pci; +// Only x86_64 is supported by percpu for now +#[cfg(CONFIG_X86_64)] +pub mod percpu; pub mod pid_namespace; pub mod platform; pub mod prelude; diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs new file mode 100644 index 000000000000..7dfceb6aefd7 --- /dev/null +++ b/rust/kernel/percpu.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0 +//! This module contains abstractions for creating and using per-CPU varia= bles from Rust. +//! See the define_per_cpu! macro and the DynamicPerCpu type, as well a= s the PerCpu trait. +pub mod cpu_guard; + +use bindings::{alloc_percpu, free_percpu}; + +use crate::alloc::Flags; +use crate::percpu::cpu_guard::CpuGuard; +use crate::prelude::*; +use crate::sync::Arc; + +use core::arch::asm; +use core::mem::{align_of, size_of}; + +use ffi::c_void; + +/// A per-CPU pointer; that is, an offset into the per-CPU area. Note that= this type is NOT a smart +/// pointer, it does not manage the allocation. +pub struct PerCpuPtr(*mut T); + +/// Represents a dynamic allocation of a per-CPU variable via alloc_percpu= . Calls free_percpu when +/// dropped. +pub struct PerCpuAllocation(PerCpuPtr); + +/// Holds a dynamically-allocated per-CPU variable. +pub struct DynamicPerCpu { + alloc: Arc>, +} + +/// Holds a statically-allocated per-CPU variable. +pub struct StaticPerCpu(PerCpuPtr); + +/// Represents exclusive access to the memory location pointed at by a par= ticular PerCpu. +pub struct PerCpuToken<'a, T> { + _guard: CpuGuard, + ptr: &'a PerCpuPtr, +} + +/// A wrapper used for declaring static per-CPU variables. These symbols a= re "virtual" in that the +/// linker uses them to generate offsets into each CPU's per-CPU area, but= shouldn't be read +/// from/written to directly. The fact that the statics are immutable prev= ents them being written +/// to (generally), this struct having _val be non-public prevents reading= from them. +/// +/// The end-user of the per-CPU API should make use of the define_per_cpu!= macro instead of +/// declaring variables of this type directly. +#[repr(transparent)] +pub struct StaticPerCpuSymbol { + _val: T, // generate a correctly sized type +} + +impl PerCpuPtr { + /// Makes a new PerCpuPtr from a raw per-CPU pointer. + /// + /// # Safety + /// `ptr` must be a valid per-CPU pointer. + pub unsafe fn new(ptr: *mut T) -> Self { + Self(ptr) + } + + /// Get a `&mut T` to the per-CPU variable represented by `&self` + /// + /// # Safety + /// The returned `&mut T` must follow Rust's aliasing rules. That is, = no other `&(mut) T` may + /// exist that points to the same location in memory. In practice, thi= s means that `get_ref` + /// must not be called on another `PerCpuPtr` that is a copy/clone = of `&self` for as long as + /// the returned reference lives. + /// + /// CPU preemption must be disabled before calling this function and f= or the lifetime of the + /// returned reference. Otherwise, the returned &mut T might end up be= ing a reference to a + /// different CPU's per-CPU area, causing the potential for a data rac= e. + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing= issues + pub unsafe fn get_ref(&self) -> &mut T { + let this_cpu_off_pcpu =3D core::ptr::addr_of!(this_cpu_off); + let mut this_cpu_area: *mut c_void; + // SAFETY: gs + this_cpu_off_pcpu is guaranteed to be a valid poin= ter because `gs` points + // to the per-CPU area and this_cpu_off_pcpu is a valid per-CPU al= location. + unsafe { + asm!( + "mov {out}, gs:[{off_val}]", + off_val =3D in(reg) this_cpu_off_pcpu, + out =3D out(reg) this_cpu_area, + ) + }; + // SAFETY: this_cpu_area + self.0 is guaranteed to be a valid poin= ter by the per-CPU + // subsystem and the invariant that self.0 is a valid offset into = the per-CPU area. + // + // We know no-one else has a reference to the underlying pcpu vari= able because of the + // safety requirements of this function. + unsafe { &mut *((this_cpu_area).wrapping_add(self.0 as usize) as *= mut T) } + } +} + +impl Clone for PerCpuPtr { + fn clone(&self) -> Self { + *self + } +} + +/// PerCpuPtr is just a pointer, so it's safe to copy. +impl Copy for PerCpuPtr {} + +impl PerCpuAllocation { + /// Dynamically allocates a space in the per-CPU area suitably sized a= nd aligned to hold a `T`. + /// + /// Returns `None` under the same circumstances the C function `alloc_= percpu` returns `NULL`. + pub fn new() -> Option> { + // SAFETY: No preconditions to call alloc_percpu + let ptr: *mut T =3D unsafe { alloc_percpu(size_of::(), align_of= ::()) } as *mut T; + if ptr.is_null() { + return None; + } + + Some(Self(PerCpuPtr(ptr))) + } +} + +impl Drop for PerCpuAllocation { + fn drop(&mut self) { + // SAFETY: self.0.0 was returned by alloc_percpu, and so was a val= id pointer into + // the percpu area, and has remained valid by the invariants of Pe= rCpuAllocation. + unsafe { free_percpu(self.0 .0 as *mut c_void) } + } +} + +/// A trait representing a per-CPU variable. This is implemented for both = `StaticPerCpu` and +/// `DynamicPerCpu`. The main usage of this trait is to call `get` to g= et a `PerCpuToken` that +/// can be used to access the underlying per-CPU variable. See `PerCpuToke= n::with`. +/// +/// # Safety +/// The returned value from `ptr` must be valid for the lifetime of `&mut = self`. +pub unsafe trait PerCpu { + /// Gets a `PerCpuPtr` to the per-CPU variable represented by `&mut= self` + /// + /// # Safety + /// `self` may be doing all sorts of things to track when the underlyi= ng per-CPU variable can + /// be deallocated. You almost certainly shouldn't be calling this fun= ction directly (it's + /// essentially an implementation detail of the trait), and you certai= nly shouldn't be making + /// copies of the returned `PerCpuPtr` that may outlive `&mut self`. + /// + /// Implementers of this trait must ensure that the returned `PerCpuPt= r` is valid for the + /// lifetime of `&mut self`. + unsafe fn ptr(&mut self) -> &PerCpuPtr; + + /// Produces a token, asserting that the holder has exclusive access t= o the underlying memory + /// pointed to by `self` + /// + /// # Safety + /// `func` (or its callees that execute on the same CPU) may not, for = any `x: PerCpu` that + /// is a `clone` of `&mut self` (or, for a statically allocated variab= le, a `StaticPerCpu` + /// that came from the same `define_per_cpu!`): + /// - call `x.get()` + /// - make use of the value returned by `x.ptr()` + /// + /// `func` and its callees must not access or modify the memory associ= ated with `&mut self`'s + /// allocation in the per-CPU area, except via (reborrows of) the refe= rence passed to `func`. + /// + /// The underlying per-CPU variable cannot ever be mutated from an int= errupt context, unless + /// irqs are disabled for the lifetime of the returned `PerCpuToken`. + unsafe fn get(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { + PerCpuToken { + _guard: guard, + // SAFETY: The lifetime of the returned `PerCpuToken<'_, T>` i= s bounded by the lifetime + // of `&mut self`. + ptr: unsafe { self.ptr() }, + } + } +} + +impl StaticPerCpu { + /// Creates a new PerCpu pointing to the statically allocated varia= ble at `ptr`. End-users + /// should probably be using the `unsafe_get_per_cpu!` macro instead o= f calling this function. + /// + /// # Safety + /// `ptr` must be a valid pointer to a per-CPU variable. This means th= at it must be a valid + /// offset into the per-CPU area, and that the per-CPU area must be su= itably sized and aligned + /// to hold a `T`. + pub unsafe fn new(ptr: *mut T) -> Self { + Self(PerCpuPtr(ptr)) + } +} + +// SAFETY: The `PerCpuPtr` returned by `ptr` is valid for the lifetime = of `self` (and in fact, +// forever). +unsafe impl PerCpu for StaticPerCpu { + unsafe fn ptr(&mut self) -> &PerCpuPtr { + &self.0 + } +} + +impl Clone for StaticPerCpu { + fn clone(&self) -> Self { + Self(self.0) + } +} + +impl DynamicPerCpu { + /// Allocates a new per-CPU variable + /// + /// # Arguments + /// * `flags` - Flags used to allocate an `Arc` that keeps track of th= e underlying + /// `PerCpuAllocation`. + pub fn new(flags: Flags) -> Option { + let alloc: PerCpuAllocation =3D PerCpuAllocation::new()?; + + let arc =3D Arc::new(alloc, flags).ok()?; + + Some(Self { alloc: arc }) + } +} + +impl DynamicPerCpu { + /// Wraps a `PerCpuAllocation` in a `PerCpu` + /// + /// # Arguments + /// * `alloc` - The allocation to use + /// * `flags` - The flags used to allocate an `Arc` that keeps track o= f the `PerCpuAllocation`. + pub fn new_from_allocation(alloc: PerCpuAllocation, flags: Flags) -= > Option { + let arc =3D Arc::new(alloc, flags).ok()?; + Some(Self { alloc: arc }) + } +} + +// SAFETY: The `PerCpuPtr` returned by `ptr` is valid for the lifetime = of `self` because we +// don't deallocate the underlying `PerCpuAllocation` until `self` is drop= ped. +unsafe impl PerCpu for DynamicPerCpu { + unsafe fn ptr(&mut self) -> &PerCpuPtr { + &self.alloc.0 + } +} + +impl Clone for DynamicPerCpu { + fn clone(&self) -> Self { + Self { + alloc: self.alloc.clone(), + } + } +} + +impl PerCpuToken<'_, T> { + /// Immediately invokes `func` with a `&mut T` that points at the unde= rlying per-CPU variable + /// that `&mut self` represents. + pub fn with(&mut self, func: U) + where + U: FnOnce(&mut T), + { + // SAFETY: The existence of a PerCpuToken means that the requireme= nts for get_ref are + // satisfied. + func(unsafe { self.ptr.get_ref() }); + } +} + +/// define_per_cpu! is analogous to the C DEFINE_PER_CPU macro in that it = lets you create a +/// statically allocated per-CPU variable. +/// +/// # Example +/// ``` +/// use kernel::define_per_cpu; +/// use kernel::percpu::StaticPerCpuSymbol; +/// +/// define_per_cpu!(pub MY_PERCPU: u64 =3D 0); +/// ``` +#[macro_export] +macro_rules! define_per_cpu { + ($vis:vis $id:ident: $ty:ty =3D $expr:expr) =3D> { + $crate::macros::paste! { + // Expand $expr outside of the unsafe block to avoid silently = allowing unsafe code to be + // used without a user-facing unsafe block + static [<__INIT_ $id>]: $ty =3D $expr; + + // SAFETY: StaticPerCpuSymbol is #[repr(transparent)], so w= e can freely convert from T + #[link_section =3D ".data..percpu"] + $vis static $id: StaticPerCpuSymbol<$ty> =3D unsafe { + core::mem::transmute::<$ty, StaticPerCpuSymbol<$ty>>([<__I= NIT_ $id>]) + }; + } + }; +} + +/// Gets a `StaticPerCpu` from a symbol declared with `define_per_cpu!`= or +/// `declare_extern_per_cpu!`. +/// +/// # Arguments +/// * `ident` - The identifier declared +/// +/// # Safety +/// `$id` must be declared with either `define_per_cpu!` or `declare_exter= n_per_cpu!`, and the +/// returned value must be stored in a `StaticPerCpu` where `T` matches= the declared type of +/// `$id`. +#[macro_export] +macro_rules! unsafe_get_per_cpu { + ($id:ident) =3D> {{ + $crate::percpu::StaticPerCpu::new((&$id) as *const _ as *mut _) + }}; +} + +/// Declares a StaticPerCpuSymbol corresponding to a per-CPU variable defi= ned in C. Be sure to read +/// the safety requirements of `PerCpu::get`. +#[macro_export] +macro_rules! declare_extern_per_cpu { + ($id:ident: $ty:ty) =3D> { + extern "C" { + static $id: StaticPerCpuSymbol<$ty>; + } + }; +} + +declare_extern_per_cpu!(this_cpu_off: u64); diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard= .rs new file mode 100644 index 000000000000..14c04b12e7f0 --- /dev/null +++ b/rust/kernel/percpu/cpu_guard.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Contains abstractions for disabling CPU preemption. See `CpuGuard`. + +/// A RAII guard for bindings::preempt_disable and bindings::preempt_enabl= e. Guarantees preemption +/// is disabled for as long as this object exists. +pub struct CpuGuard { + // Don't make one without using new() + _phantom: (), +} + +impl CpuGuard { + /// Create a new CpuGuard. Disables preemption for its lifetime. + pub fn new() -> Self { + // SAFETY: There are no preconditions required to call preempt_dis= able + unsafe { + bindings::preempt_disable(); + } + CpuGuard { _phantom: () } + } +} + +impl Default for CpuGuard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for CpuGuard { + fn drop(&mut self) { + // SAFETY: There are no preconditions required to call preempt_ena= ble + unsafe { + bindings::preempt_enable(); + } + } +} --=20 2.34.1 From nobody Tue Oct 7 06:50:55 2025 Received: from mail-pg1-f175.google.com (mail-pg1-f175.google.com [209.85.215.175]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7D2C5228C9D; Sat, 12 Jul 2025 21:32:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.215.175 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355928; cv=none; b=F2MlpkkMU+YgSS/wCHEDCx29QlOUjLAm02BKJgiydn5FkRYEK5Nrqe4hncLXG8hXHefpnPJnVn42OMfWSj6O1N41Xj3ZO6hi38E3XSgiNnxDylDrZAF16gm+WQerUNHmJSscVOfd6/IWS8ErQ+LlmFb4qnCjfbDzUmXI16t299c= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355928; c=relaxed/simple; bh=AP8shLBDvKbYuth+8++0JymfuCshZQX8ezE8AjEF8do=; h=From:Date:Subject:MIME-Version:Content-Type:Message-Id:References: In-Reply-To:To:Cc; b=FFlq+SkCa6ZnmYQ9ukbLTzKr+WbBM+o1rCJQPuyfHaBBuCY8VTdMAtxfh5cZRakLgpWQWI3r2PH0ADv1zl17R6V/mF96mTAZMyaEjtMuwNdzqQ7NRUu4T0EHtWAwchYIZsXNWoFZBfkbKxjojdwcnjyF2dPtTBU4MsIGAXpj+8Q= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=XcfP4Izk; arc=none smtp.client-ip=209.85.215.175 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="XcfP4Izk" Received: by mail-pg1-f175.google.com with SMTP id 41be03b00d2f7-879d2e419b9so2398627a12.2; Sat, 12 Jul 2025 14:32:07 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1752355927; x=1752960727; darn=vger.kernel.org; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:from:to:cc:subject:date:message-id :reply-to; bh=LO0TxQnKvrLipyxCSYXSthueOvIoODhR5Ygf1iirZRk=; b=XcfP4Izkdds3YAdF5ndogxAS3YmR0BfSmdft/n17dIC0/xecJYyUnZnhV70Jd6wJS9 cMuVfgVlz4AfRSFJuAlAbpZpeXAnUNyD04yl1PjdUAeXLOc3FwqR6QhjG+YLO1e5eUer ZE3iIG10BoBJh16VUbtE8QM6yVQbKWOicoxe9SKF2NpGPHmzYbsB6zamq+cCFtuT1mmC YQLY4fTKCUgn02NJyReKmeoFvYaLa2F8ZjVLpcbRZPy2xDdn3i6Tz3oMSZSwcaTc19Oo pUqrFx8aMA6JIvVvH3iEaRaMoW8liZv+TPkB+FP0Z4nG/ccEt9utM7NYztDeLMlxkZrP ml4A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752355927; x=1752960727; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=LO0TxQnKvrLipyxCSYXSthueOvIoODhR5Ygf1iirZRk=; b=Sqz4xQV+hBygMEU58h+yY9oyBh4r1EZ4JWgwGpRM4Orj2gzcm+o3Wrc+x3oa+HRsgJ 8bcWztB9gMpzoQNYGh/pt4ihHqK7HaFzNMEtTsFLc6IdRTbJUtgp+Co2o8n/px+d3BJf QOGrGXr3ZsfQfnUnF2OquUqfNM/XfoF35nV6PP6aVnI8EKo79XSn3tlSwT55e+W9A3FW tbbFttmRJsHoMR8hDeGa3N4AigG2PKKQSDtA+Ax+glppqnIizZoe4W8X8JSpBuUCzwn2 i+rN1ynuwXE9kcFJVO5SbLZD87KG0Qc0LTnQvyK6M2A2ZbfjgcQbzKWGEnkw/gxv/K1J TnPA== X-Forwarded-Encrypted: i=1; AJvYcCWxB9fkNENPnXbZ6wH4rr3aMVtc2/C03QMyPzCsixB2Z4KPfa495HOgFxuc1yZ1wC+DbHyecgWhxzLe9ULjew==@vger.kernel.org X-Gm-Message-State: AOJu0YwO1YiOnS/joE3qi+F4qXuaDJoVXW3oTknpgnhC+hTAonW4j/yS x0zzCYT1SnFEXS7VDvaLSugnmFOa0/EFCg3UfTFaCaQqZ2itgnkAjdGa X-Gm-Gg: ASbGncu1zaLlwXdni24SarpnwSoW40wguzABZJqq9lpYt/AsRPs6Fm4kPWujSYvD21U YDzfQa/wexh7aKQKMjONtbbKiCObJ4RD24/Txuz5K2GDyb1eNzqBrQArLjeoFFFPuaobQMvFgYB jXTkCJIW5AnxAwAMxFEq1mVS8xyKKmfl+vGs7yyKCAQj/bI9I9VCGJMPtY8GHpCPI4tvp3/dvyZ r98R3jIawRgCecJpo0hySrFLJ+qZ4t285mmao+bF6XYtHa/FN9bFqEQCm1iN7kmqPwdnnueeMot xiOVdppoQDkCoN/OjcpbROp5DuoDUggXiOprkBNPH+Y5U0Mv2RGyq/c1Q2s4BGg7mZE/tBIbl9U nuG8mqkpFfv6jpPjAvSOKEWCLKWT0PIMVTCJYsXc7YsucEWD8ttwZHQz7 X-Google-Smtp-Source: AGHT+IFPX3vsrSmFB/bzkd4mz1ZHYM0i6Eaod7AfCsmL0k7n193sAJy+iT0OV/FyWJP513vCy+Nl8w== X-Received: by 2002:a17:90b:5101:b0:312:db8:dbd2 with SMTP id 98e67ed59e1d1-31c4cd704f5mr11477475a91.19.1752355926653; Sat, 12 Jul 2025 14:32:06 -0700 (PDT) Received: from mitchelllevy.localdomain ([174.127.224.194]) by smtp.gmail.com with ESMTPSA id 98e67ed59e1d1-31c3eb65aa2sm8003935a91.40.2025.07.12.14.32.05 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 12 Jul 2025 14:32:06 -0700 (PDT) From: Mitchell Levy Date: Sat, 12 Jul 2025 14:31:13 -0700 Subject: [PATCH v2 2/5] rust: rust-analyzer: add lib to dirs searched for crates Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Message-Id: <20250712-rust-percpu-v2-2-826f2567521b@gmail.com> References: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> In-Reply-To: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> To: Miguel Ojeda , Alex Gaynor , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Andreas Hindborg , Alice Ryhl , Trevor Gross , Andrew Morton , Dennis Zhou , Tejun Heo , Christoph Lameter , Danilo Krummrich , Benno Lossin Cc: linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-mm@kvack.org, Mitchell Levy X-Mailer: b4 0.14.0 X-Developer-Signature: v=1; a=ed25519-sha256; t=1752355923; l=925; i=levymitchell0@gmail.com; s=20240719; h=from:subject:message-id; bh=AP8shLBDvKbYuth+8++0JymfuCshZQX8ezE8AjEF8do=; b=03+IgQe2BFqZTMGtBiBKVhcvABMJFgRtkYVcIW8McilA/S6CB3mUuXaqWNv0d/2AtfLUUb8NW +4W5XFT/fdbCnKpZ5UQXgRCf/5Y9/HTZgnV27ZNcBXSB/U9nWlpmWkc X-Developer-Key: i=levymitchell0@gmail.com; a=ed25519; pk=n6kBmUnb+UNmjVkTnDwrLwTJAEKUfs2e8E+MFPZI93E= When generating rust-project.json, also include crates in lib/ Signed-off-by: Mitchell Levy --- scripts/generate_rust_analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_anal= yzer.py index 7c3ea2b55041..08e14ae9c1a0 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -152,7 +152,7 @@ def generate_crates(srctree, objtree, sysroot_src, exte= rnal_src, cfgs, core_edit # Then, the rest outside of `rust/`. # # We explicitly mention the top-level folders we want to cover. - extra_dirs =3D map(lambda dir: srctree / dir, ("samples", "drivers")) + extra_dirs =3D map(lambda dir: srctree / dir, ("samples", "drivers", "= lib")) if external_src is not None: extra_dirs =3D [external_src] for folder in extra_dirs: --=20 2.34.1 From nobody Tue Oct 7 06:50:55 2025 Received: from mail-pl1-f181.google.com (mail-pl1-f181.google.com [209.85.214.181]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E98A222B5A3; Sat, 12 Jul 2025 21:32:08 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.214.181 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355930; cv=none; b=FQbc1mdCtXM5SgZqS6kamskq2I10zS3AfQy0q41WTzMbur4ATX2ZAKwYRY68N7NqALMOeHzzCqRIH5eSv49v+ou/uRyrZZxrzbMShkHqqGYJl+TC+I8iu/ELXLgg/jsa9LDqgP7ujaxafLGU+yoJ54ap/157yqxUmHgDd2zimrk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355930; c=relaxed/simple; bh=J2wN034E1ANJf8z7zTgfPajB+DTsr36981US4k79sBA=; h=From:Date:Subject:MIME-Version:Content-Type:Message-Id:References: In-Reply-To:To:Cc; b=XCR1VM2o+N42j0vYuFrT32MR2pS/BMxypzdm2xERaBRqKgMAaDT8QA1rzZ/Vu40Ai+PaXqgjj9DEeaAHev7LJ/lYmU0KO4GlM6qnmlvpgccnSu+E1i1nF/mpWYzMCLGlxIQ5PWQRtpEhkgYikGOxccPGxO415BNHOAqmkA4vpM0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=S4AMNHVM; arc=none smtp.client-ip=209.85.214.181 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="S4AMNHVM" Received: by mail-pl1-f181.google.com with SMTP id d9443c01a7336-235f9e87f78so31024085ad.2; Sat, 12 Jul 2025 14:32:08 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1752355928; x=1752960728; darn=vger.kernel.org; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:from:to:cc:subject:date:message-id :reply-to; bh=hExQIcvoKTg+0rOcrZXWVdh84oK5KeLOdM+0A3CpbHg=; b=S4AMNHVMRb+1ydb3534aeTmdf/Sw0xHKhv53kjIpYUaejm9PG7xTA81qwL2eCKEH0L HNf1dmaiHqw/OxbcrjZhe/HG+w0H6nZtcYMdb0WfwIaB7tEnk4tN/wWjg9GjDqZraWOl aYwU2uIxItO8dEJf2+zjBcyUleFb6VwLcODoVkNt18I9tQkDdLW1CmKT1QrMFycE03UO RMGtu35lcZC8vw2YtKEuIDMxeJWAJXs2V8BHncHWOpK1dYn7erbAEl2/pI5TUfGQCM0J 3t1OJK/+0oMNnaTWmLmzP7B9BJJqGGhCYfcdSHG+PdMp5a/Mjq8GlbwEgDZ8GBO0JHN2 tERg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752355928; x=1752960728; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=hExQIcvoKTg+0rOcrZXWVdh84oK5KeLOdM+0A3CpbHg=; b=MDfB6xzpvGUGNII9kmZKw2kaYqX/C53wP6A3eR5+QXkYNwWLRHlM+3u+jGynA1yDu8 sAtbJaOMFTwCbFwwi7aFOWeYTbCJTVLxJJWmZypcAzToQFD9fmCcn0muBP0/CN+/8nMn pTtUW8k4BF6BogmRBYnaeTSJAi/+fRi1+YUn+fJJ3e7qlEM+37pa/YOtk2zBavTRrdNb 9Vz7aBS3Sz/gNwojOvZ4Z2iVyeVG+6osw53IxxgIXQkWMvjMETwmhhhX5q5DwXA3XM1T /l9X4S6V79Tr31s5gtr3O5siIEK9J5nhwNXm7KzjYHouSn9oFWIV1U3KU4Uls65dfb+r /5CQ== X-Forwarded-Encrypted: i=1; AJvYcCXCnjXo4xdUURGmxtUnV0KhOOr+xT8PQnJPPwYlPzuPwjYdaulmx4xFZRReflHzHoGS9/UrspROljM8C/8MJg==@vger.kernel.org X-Gm-Message-State: AOJu0YzmzX0GRn/6X2r93WNJUo0ErGIvF39na7fg+6oBZB71jkQRT6Qq p1BosyMOmwTo81geSmVNQf9ZUzwqXeTpcwFEMGpRReuqFQ9xw2frB2MK X-Gm-Gg: ASbGncsUjqAD2w2JQk8b5GyIuydVMg/vpf8v34WjNVu4eeMS8AjbSspV2FHqUQE5iDV BLhhYFgws4Nr76QzaCS08ueuV3tlfyPI8FFGdGtFTkCtQskupK0/OJG7TOVhMOs3W9aQp/mK8aJ Tf2zuqSLjb4/RU3WKeZ2e6ptEwLCaA33Yk9ogHdXYcxzEj0GdtiEAhMCeFH7adp1OY9A5Cvkf3N r+FaiCwjLJqXdtR4z68giTLrxXq/GVOqogIGfLGTuOnszr3PWeYr7p63bL8mq6vrUUEmxwPM3WB lTd92UPkpwBy0Lzgcfak7lKmoqPFm2f98g2WFPtu/LxIYrtxbsZqXzB1PT3Vis9nY1xV1dJIzh1 h65nUuITjtMDqGRyVCsaWNGqRfP9o7UhjDvUDFZ0n9jB8AQ== X-Google-Smtp-Source: AGHT+IG61yxEcItnZRLBCpzRu1bY9moREdTEBhg3R8F8LHZq7g2AdMJ+EkH1Plxdpo90qEBetLdPOQ== X-Received: by 2002:a17:903:a8f:b0:22e:5d9b:2ec3 with SMTP id d9443c01a7336-23dede7d432mr121624255ad.30.1752355927891; Sat, 12 Jul 2025 14:32:07 -0700 (PDT) Received: from mitchelllevy.localdomain ([174.127.224.194]) by smtp.gmail.com with ESMTPSA id 98e67ed59e1d1-31c3eb65aa2sm8003935a91.40.2025.07.12.14.32.06 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 12 Jul 2025 14:32:07 -0700 (PDT) From: Mitchell Levy Date: Sat, 12 Jul 2025 14:31:14 -0700 Subject: [PATCH v2 3/5] rust: percpu: add a rust per-CPU variable test Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Message-Id: <20250712-rust-percpu-v2-3-826f2567521b@gmail.com> References: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> In-Reply-To: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> To: Miguel Ojeda , Alex Gaynor , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Andreas Hindborg , Alice Ryhl , Trevor Gross , Andrew Morton , Dennis Zhou , Tejun Heo , Christoph Lameter , Danilo Krummrich , Benno Lossin Cc: linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-mm@kvack.org, Mitchell Levy X-Mailer: b4 0.14.0 X-Developer-Signature: v=1; a=ed25519-sha256; t=1752355923; l=6875; i=levymitchell0@gmail.com; s=20240719; h=from:subject:message-id; bh=J2wN034E1ANJf8z7zTgfPajB+DTsr36981US4k79sBA=; b=ujc/F9znVc9J+1BY7Iu7YhgYa/WWZ1LKlVHVug/z90X/Ix09YeqHueTf5rN3LoU+an3ffOqsi Dl+ePBIO/+ICPEQO8VX7uxHlAnutjF0PL44suFsx/P1hiIJUkvPQVaA X-Developer-Key: i=levymitchell0@gmail.com; a=ed25519; pk=n6kBmUnb+UNmjVkTnDwrLwTJAEKUfs2e8E+MFPZI93E= Add a short exercise for Rust's per-CPU variable API, modelled after lib/percpu_test.c Signed-off-by: Mitchell Levy --- lib/Kconfig.debug | 9 ++++ lib/Makefile | 1 + lib/percpu_test_rust.rs | 120 ++++++++++++++++++++++++++++++++++++++++++++= ++++ rust/helpers/percpu.c | 11 +++++ 4 files changed, 141 insertions(+) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index ebe33181b6e6..959ce156c601 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2418,6 +2418,15 @@ config PERCPU_TEST =20 If unsure, say N. =20 +config PERCPU_TEST_RUST + tristate "Rust per cpu operations test" + depends on m && DEBUG_KERNEL && RUST + help + Enable this option to build a test module which validates Rust per-cpu + operations. + + If unsure, say N. + config ATOMIC64_SELFTEST tristate "Perform an atomic64_t self-test" help diff --git a/lib/Makefile b/lib/Makefile index c38582f187dd..ab19106cc22c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -288,6 +288,7 @@ obj-$(CONFIG_RBTREE_TEST) +=3D rbtree_test.o obj-$(CONFIG_INTERVAL_TREE_TEST) +=3D interval_tree_test.o =20 obj-$(CONFIG_PERCPU_TEST) +=3D percpu_test.o +obj-$(CONFIG_PERCPU_TEST_RUST) +=3D percpu_test_rust.o =20 obj-$(CONFIG_ASN1) +=3D asn1_decoder.o obj-$(CONFIG_ASN1_ENCODER) +=3D asn1_encoder.o diff --git a/lib/percpu_test_rust.rs b/lib/percpu_test_rust.rs new file mode 100644 index 000000000000..a9652e6ece08 --- /dev/null +++ b/lib/percpu_test_rust.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +//! A simple self test for the rust per-CPU API. + +use core::ffi::c_void; + +use kernel::{ + bindings::{on_each_cpu, smp_processor_id}, + define_per_cpu, + percpu::{cpu_guard::*, *}, + pr_info, + prelude::*, + unsafe_get_per_cpu, +}; + +module! { + type: PerCpuTestModule, + name: "percpu_test_rust", + author: "Mitchell Levy", + description: "Test code to exercise the Rust Per CPU variable API", + license: "GPL v2", +} + +struct PerCpuTestModule; + +define_per_cpu!(PERCPU: i64 =3D 0); +define_per_cpu!(UPERCPU: u64 =3D 0); + +impl kernel::Module for PerCpuTestModule { + fn init(_module: &'static ThisModule) -> Result { + pr_info!("rust percpu test start\n"); + + let mut native: i64 =3D 0; + // SAFETY: PERCPU is properly defined + let mut pcpu: StaticPerCpu =3D unsafe { unsafe_get_per_cpu!(P= ERCPU) }; + // SAFETY: We only have one PerCpu that points at PERCPU + unsafe { pcpu.get(CpuGuard::new()) }.with(|val: &mut i64| { + pr_info!("The contents of pcpu are {}\n", val); + + native +=3D -1; + *val +=3D -1; + pr_info!("Native: {}, *pcpu: {}\n", native, val); + assert!(native =3D=3D *val && native =3D=3D -1); + + native +=3D 1; + *val +=3D 1; + pr_info!("Native: {}, *pcpu: {}\n", native, val); + assert!(native =3D=3D *val && native =3D=3D 0); + }); + + let mut unative: u64 =3D 0; + // SAFETY: UPERCPU is properly defined + let mut upcpu: StaticPerCpu =3D unsafe { unsafe_get_per_cpu!(= UPERCPU) }; + + // SAFETY: We only have one PerCpu pointing at UPERCPU + unsafe { upcpu.get(CpuGuard::new()) }.with(|val: &mut u64| { + unative +=3D 1; + *val +=3D 1; + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative =3D=3D *val && unative =3D=3D 1); + + unative =3D unative.wrapping_add((-1i64) as u64); + *val =3D val.wrapping_add((-1i64) as u64); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative =3D=3D *val && unative =3D=3D 0); + + unative =3D unative.wrapping_add((-1i64) as u64); + *val =3D val.wrapping_add((-1i64) as u64); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative =3D=3D *val && unative =3D=3D (-1i64) as u64); + + unative =3D 0; + *val =3D 0; + + unative =3D unative.wrapping_sub(1); + *val =3D val.wrapping_sub(1); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative =3D=3D *val && unative =3D=3D (-1i64) as u64); + assert!(unative =3D=3D *val && unative =3D=3D u64::MAX); + }); + + pr_info!("rust static percpu test done\n"); + + pr_info!("rust dynamic percpu test start\n"); + let mut test: DynamicPerCpu =3D DynamicPerCpu::new(GFP_KERNEL= ).unwrap(); + + // SAFETY: No prerequisites for on_each_cpu. + unsafe { + on_each_cpu(Some(inc_percpu), (&raw mut test) as *mut c_void, = 0); + on_each_cpu(Some(inc_percpu), (&raw mut test) as *mut c_void, = 0); + on_each_cpu(Some(inc_percpu), (&raw mut test) as *mut c_void, = 0); + on_each_cpu(Some(inc_percpu), (&raw mut test) as *mut c_void, = 1); + on_each_cpu(Some(check_percpu), (&raw mut test) as *mut c_void= , 1); + } + + pr_info!("rust dynamic percpu test done\n"); + + // Return Err to unload the module + Result::Err(EINVAL) + } +} + +extern "C" fn inc_percpu(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu and D= ynamicPerCpu is Send. + let mut pcpu =3D unsafe { (*(info as *const DynamicPerCpu)).clone= () }; + // SAFETY: smp_processor_id has no preconditions + pr_info!("Incrementing on {}\n", unsafe { smp_processor_id() }); + + // SAFETY: We don't have multiple clones of pcpu in scope + unsafe { pcpu.get(CpuGuard::new()) }.with(|val: &mut u64| *val +=3D 1); +} + +extern "C" fn check_percpu(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu and D= ynamicPerCpu is Send. + let mut pcpu =3D unsafe { (*(info as *const DynamicPerCpu)).clone= () }; + // SAFETY: smp_processor_id has no preconditions + pr_info!("Asserting on {}\n", unsafe { smp_processor_id() }); + + // SAFETY: We don't have multiple clones of pcpu in scope + unsafe { pcpu.get(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val= =3D=3D 4)); +} diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c index a091389f730f..0e9b2fed3ebd 100644 --- a/rust/helpers/percpu.c +++ b/rust/helpers/percpu.c @@ -1,9 +1,20 @@ // SPDX-License-Identifier: GPL-2.0 =20 #include +#include =20 void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) { return __alloc_percpu(sz, align); } =20 +void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait) +{ + on_each_cpu(func, info, wait); +} + +int rust_helper_smp_processor_id(void) +{ + return smp_processor_id(); +} + --=20 2.34.1 From nobody Tue Oct 7 06:50:55 2025 Received: from mail-pj1-f51.google.com (mail-pj1-f51.google.com [209.85.216.51]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 88BFF22F16C; Sat, 12 Jul 2025 21:32:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.216.51 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355932; cv=none; b=T2cSUgVVqGAAmyQOb7GV5riRQ1tvPHHRa7kmbdff8WSA4QAXsD5vtZ0otY2EujizZq+WBDJxP2pjPI8kN8MTEiVVjAh1lYoC/VLpvXn21mzZKKCU5uekirdmzHaH0TJARLTPWs2xUAz/6jS5a4LPC8Ydx0JJ5HYTNj8KxGHcqHQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355932; c=relaxed/simple; bh=Yhc2AZNWstxq/JmR6oLcJC1JQH1zZ60L+8ZJ13d/Yes=; h=From:Date:Subject:MIME-Version:Content-Type:Message-Id:References: In-Reply-To:To:Cc; b=sWku+16VaYVSVJZq6zVS+8e0Wo9Atmbz97Zt+p0lX7rn8l/eyUyi3kogmbnkfgCTq8AbQTQqX+O404ItmAdXwjbqlSqifcAx/JA5AdbyGi3uQDQqotmObARIbckcHBTlgT3ja2LTdKjxzD8/dX5xTQafawPjl6uieehC9MXNouo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=iWV9vpek; arc=none smtp.client-ip=209.85.216.51 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="iWV9vpek" Received: by mail-pj1-f51.google.com with SMTP id 98e67ed59e1d1-313eeb77b1fso2719187a91.1; Sat, 12 Jul 2025 14:32:09 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1752355929; x=1752960729; darn=vger.kernel.org; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:from:to:cc:subject:date:message-id :reply-to; bh=7j+nFClWMYu+z4RxyaBNy1i8eSeqUJQPpZ3XTZUjlQs=; b=iWV9vpekQ6CHjffOM4MnleRPqDyO/F6tvsuWh146EKFndw+rjulJKmoGdpW4lAft+E QvQqT8F+GgxxYWx5lguD5Lcn7JDJ6i8yphB+yf55f0lusHbitPtHxCm313Skb8WtGhgL vymKSzMPSYQ2LM2TsI3QWHvBYDt4kBAQGOLisONu/X7Ikfydpd3i4lukzZGzO+58Gtn9 tKrh4xE+f17Cue2IBLztHhl1B4Iek7Pe4Z9Te5Z4TzAqigEQ0a4vz8uyacJz6Ieh5h4g rGpGtTAneRQ6jRRbXuzybvjK3vpJFQ6zvJT8g95UKkQ0Z8gemb0VNl/syh3bl1X9p1Dp aZ+Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752355929; x=1752960729; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=7j+nFClWMYu+z4RxyaBNy1i8eSeqUJQPpZ3XTZUjlQs=; b=QhX1svdStuOoA1Ne5/7L7Sq39rQfRggNErAE2HnOIZgZGhVPxcR3hN4ekBDiOnBx8y G8S7/HQ2QKoDt81+jy8thI2AsS9y9gS6HOjF0ee9qRDIgIypiFiWzafMWxPjZCOdhqNT rKJJ5NSpKPM+JwnnNJwIEQFXzjz6OLmE44cbf+VYX5EGfIsX2BShuWH3USc86AjJWAc9 eBptKgC45qf+rz47PtY6K8A8USnB0z9pq4nHCBakuvLJACPWLC5Tiid+6mYkRgHI91SJ AnV+T1YyUMWYhrbBcZurRHlPVNCnacTgUPvaZlnsD5xhB0WQ52B7QTOpKCZMhOfMzwKk 8bPQ== X-Forwarded-Encrypted: i=1; AJvYcCWgtOBCA/b8UNicE/zmQDVFI/lqixx54oqB3WJyBog6WAD4QNTCsvnnYWCPhUeQXf3R1TePeGiCL5J9kNg6/Q==@vger.kernel.org X-Gm-Message-State: AOJu0YwQtIUNczzDmZwLN8SOv228eZmsgrl8vbIS6kvty8OWVhIuZtGM F8anLJZL0A3pM9g3N1RVqMVUk6ppdYcoE2BD2CWoB/sUW+DFYK0l9GY7MyHA53xeT50= X-Gm-Gg: ASbGncvtZ6JEV/PCeW1fMldKY5Ww8mKyGGCQdY1goO2NeRWuQC8/5y98cx7d/lSgadv zaetkJnjslPoFCJtjOuwuqnEOjATLo09z4YLIm7hwlgqOl07flxPfNg+AJpqz+oDZjjdKPA4So+ 0PBj5XJ5xLrU3Q0K1yYSnlq/peVrO8JxbUeiWFs/sZxbcAyk+6G2m9Kc9vq3bk9WMshfRTp9oBi rCXoBR7WCc6oiJUnFXDApFSbi1S2NbwuT8FDiZCpxSsBY0HCpPAp9HmSeF78w+iSOdrL/2fN1PV TH8CU4i5oNqr5qMK9zB4AAwVTufg40Cm70aYzUJaArJfXgK4OjHi0HLE/XEHPvjUDAVtbaMxgMZ Rp2+1XODv5aMPCXQvmXdtVqR0bSyicwRAAvOWPt53zWGcVg== X-Google-Smtp-Source: AGHT+IE4OVGv0ZHONTY8jo3BV9GHDYUu3GtoFhSxfwHUuz7H8XXTW3Jjoc+7pdproO8lCLYVAmLqDw== X-Received: by 2002:a17:90b:3503:b0:31c:260e:5603 with SMTP id 98e67ed59e1d1-31c4f562f15mr11707603a91.24.1752355928711; Sat, 12 Jul 2025 14:32:08 -0700 (PDT) Received: from mitchelllevy.localdomain ([174.127.224.194]) by smtp.gmail.com with ESMTPSA id 98e67ed59e1d1-31c3eb65aa2sm8003935a91.40.2025.07.12.14.32.08 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 12 Jul 2025 14:32:08 -0700 (PDT) From: Mitchell Levy Date: Sat, 12 Jul 2025 14:31:15 -0700 Subject: [PATCH v2 4/5] rust: percpu: Add pin-hole optimizations for numerics Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Message-Id: <20250712-rust-percpu-v2-4-826f2567521b@gmail.com> References: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> In-Reply-To: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> To: Miguel Ojeda , Alex Gaynor , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Andreas Hindborg , Alice Ryhl , Trevor Gross , Andrew Morton , Dennis Zhou , Tejun Heo , Christoph Lameter , Danilo Krummrich , Benno Lossin Cc: linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-mm@kvack.org, Mitchell Levy X-Mailer: b4 0.14.0 X-Developer-Signature: v=1; a=ed25519-sha256; t=1752355923; l=8164; i=levymitchell0@gmail.com; s=20240719; h=from:subject:message-id; bh=Yhc2AZNWstxq/JmR6oLcJC1JQH1zZ60L+8ZJ13d/Yes=; b=7PlUlkr00EGSTFBjGecI5Y1jCbBQRf4K+CpEu8vYyLOv26zqXz/i9wBlJRC3sha4iMw1CGAtR dmDSdeDAdtXCDQ1FfikH92Lsn9VQUB/B0uiLGstCcKlAmie94nNufey X-Developer-Key: i=levymitchell0@gmail.com; a=ed25519; pk=n6kBmUnb+UNmjVkTnDwrLwTJAEKUfs2e8E+MFPZI93E= The C implementations of `this_cpu_add`, `this_cpu_sub`, etc., are optimized to save an instruction by avoiding having to compute `this_cpu_ptr(&x)` for some per-CPU variable `x`. For example, rather than u64 *x_ptr =3D this_cpu_ptr(&x); *x_ptr +=3D 5; the implementation of `this_cpu_add` is clever enough to make use of the fact that per-CPU variables are implemented on x86 via segment registers, and so we can use only a single instruction (where we assume `&x` is already in `rax`) add gs:[rax], 5 Add this optimization via a `PerCpuNumeric` type to enable code-reuse between `DynamicPerCpu` and `StaticPerCpu`. Signed-off-by: Mitchell Levy --- lib/percpu_test_rust.rs | 36 +++++++++++++ rust/kernel/percpu.rs | 1 + rust/kernel/percpu/numeric.rs | 117 ++++++++++++++++++++++++++++++++++++++= ++++ 3 files changed, 154 insertions(+) diff --git a/lib/percpu_test_rust.rs b/lib/percpu_test_rust.rs index a9652e6ece08..114015435a85 100644 --- a/lib/percpu_test_rust.rs +++ b/lib/percpu_test_rust.rs @@ -25,6 +25,26 @@ define_per_cpu!(PERCPU: i64 =3D 0); define_per_cpu!(UPERCPU: u64 =3D 0); =20 +macro_rules! make_optimization_test { + ($ty:ty) =3D> { + let mut test: DynamicPerCpu<$ty> =3D DynamicPerCpu::new(GFP_KERNEL= ).unwrap(); + { + let _ =3D CpuGuard::new(); + // SAFETY: No other usage of `test` + unsafe { test.get(CpuGuard::new()) }.with(|val: &mut $ty| *val= =3D 10); + test.num().add(1); + // SAFETY: No other usage of `test` + unsafe { test.get(CpuGuard::new()) }.with(|val: &mut $ty| asse= rt_eq!(*val, 11)); + test.num().add(10); + // SAFETY: No other usage of `test` + unsafe { test.get(CpuGuard::new()) }.with(|val: &mut $ty| asse= rt_eq!(*val, 21)); + test.num().sub(5); + // SAFETY: No other usage of `test` + unsafe { test.get(CpuGuard::new()) }.with(|val: &mut $ty| asse= rt_eq!(*val, 16)); + } + }; +} + impl kernel::Module for PerCpuTestModule { fn init(_module: &'static ThisModule) -> Result { pr_info!("rust percpu test start\n"); @@ -94,6 +114,22 @@ fn init(_module: &'static ThisModule) -> Result { =20 pr_info!("rust dynamic percpu test done\n"); =20 + pr_info!("rust numeric optimizations test start\n"); + + make_optimization_test!(u8); + make_optimization_test!(u16); + make_optimization_test!(u32); + make_optimization_test!(u64); + make_optimization_test!(usize); + + make_optimization_test!(i8); + make_optimization_test!(i16); + make_optimization_test!(i32); + make_optimization_test!(i64); + make_optimization_test!(isize); + + pr_info!("rust numeric optimizations test done\n"); + // Return Err to unload the module Result::Err(EINVAL) } diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs index 7dfceb6aefd7..b97d1d07a614 100644 --- a/rust/kernel/percpu.rs +++ b/rust/kernel/percpu.rs @@ -2,6 +2,7 @@ //! This module contains abstractions for creating and using per-CPU varia= bles from Rust. //! See the define_per_cpu! macro and the DynamicPerCpu type, as well a= s the PerCpu trait. pub mod cpu_guard; +pub mod numeric; =20 use bindings::{alloc_percpu, free_percpu}; =20 diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs new file mode 100644 index 000000000000..e4008f872af1 --- /dev/null +++ b/rust/kernel/percpu/numeric.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Pin-hole optimizations for PerCpu where T is a numeric type. + +use crate::percpu::*; +use core::arch::asm; + +/// Represents a per-CPU variable that can be manipulated with machine-int= rinsic numeric +/// operations. +pub struct PerCpuNumeric<'a, T> { + ptr: &'a PerCpuPtr, +} + +macro_rules! impl_ops { + ($ty:ty, $reg:tt) =3D> { + impl DynamicPerCpu<$ty> { + /// Returns a `PerCpuNumeric` that can be used to manipulate t= he underlying per-CPU variable. + pub fn num(&self) -> PerCpuNumeric<'_, $ty> { + PerCpuNumeric { ptr: &self.alloc.0 } + } + } + impl StaticPerCpu<$ty> { + /// Returns a `PerCpuNumeric` that can be used to manipulate t= he underlying per-CPU variable. + pub fn num(&self) -> PerCpuNumeric<'_, $ty> { + PerCpuNumeric { ptr: &self.0 } + } + } + + impl PerCpuNumeric<'_, $ty> { + /// Adds `rhs` to the per-CPU variable. + pub fn add(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU= area (i.e., valid as a + // pointer relative to the `gs` segment register) by the i= nvariants of PerCpu. + unsafe { + asm!( + concat!("add gs:[{off}], {val:", $reg, "}"), + off =3D in(reg) self.ptr.0 as *mut $ty, + val =3D in(reg) rhs, + ); + } + } + } + impl PerCpuNumeric<'_, $ty> { + /// Subtracts `rhs` from the per-CPU variable. + pub fn sub(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU= area (i.e., valid as a + // pointer relative to the `gs` segment register) by the i= nvariants of PerCpu. + unsafe { + asm!( + concat!("sub gs:[{off}], {val:", $reg, "}"), + off =3D in(reg) self.ptr.0 as *mut $ty, + val =3D in(reg) rhs, + ); + } + } + } + }; +} + +macro_rules! impl_ops_byte { + ($ty:ty) =3D> { + impl DynamicPerCpu<$ty> { + /// Returns a `PerCpuNumeric` that can be used to manipulate t= he underlying per-CPU + /// variable. + pub fn num(&self) -> PerCpuNumeric<'_, $ty> { + PerCpuNumeric { ptr: &self.alloc.0 } + } + } + impl StaticPerCpu<$ty> { + /// Returns a `PerCpuNumeric` that can be used to manipulate t= he underlying per-CPU + /// variable. + pub fn num(&self) -> PerCpuNumeric<'_, $ty> { + PerCpuNumeric { ptr: &self.0 } + } + } + + impl PerCpuNumeric<'_, $ty> { + /// Adds `rhs` to the per-CPU variable. + pub fn add(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU= area (i.e., valid as a + // pointer relative to the `gs` segment register) by the i= nvariants of PerCpu. + unsafe { + asm!( + concat!("add gs:[{off}], {val}"), + off =3D in(reg) self.ptr.0 as *mut $ty, + val =3D in(reg_byte) rhs, + ); + } + } + } + impl PerCpuNumeric<'_, $ty> { + /// Subtracts `rhs` from the per-CPU variable. + pub fn sub(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU= area (i.e., valid as a + // pointer relative to the `gs` segment register) by the i= nvariants of PerCpu. + unsafe { + asm!( + concat!("sub gs:[{off}], {val}"), + off =3D in(reg) self.ptr.0 as *mut $ty, + val =3D in(reg_byte) rhs, + ); + } + } + } + }; +} + +impl_ops_byte!(i8); +impl_ops!(i16, "x"); +impl_ops!(i32, "e"); +impl_ops!(i64, "r"); +impl_ops!(isize, "r"); + +impl_ops_byte!(u8); +impl_ops!(u16, "x"); +impl_ops!(u32, "e"); +impl_ops!(u64, "r"); +impl_ops!(usize, "r"); --=20 2.34.1 From nobody Tue Oct 7 06:50:55 2025 Received: from mail-pj1-f50.google.com (mail-pj1-f50.google.com [209.85.216.50]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7287A22F389; Sat, 12 Jul 2025 21:32:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.216.50 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355932; cv=none; b=AXpy5pXffQc3gO3l7yHrkK3Y1CHcuGOOb7S4ZPW4Uuljo9U6WUge8lYbof6TwOhlhHnKabmMc2KYQL19oxyeF9a8uCuIYLWkHnuRIxHS6cF5qjxhcczxoclGkelhxCl9VorgxsGoUhnoapL0UnQOD8J02CAw6omQpDs8tH/l2Mw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752355932; c=relaxed/simple; bh=e1kjOZMlHir0QAwDPK2Mh6knOwIrBRKVgMEnFQ0sips=; h=From:Date:Subject:MIME-Version:Content-Type:Message-Id:References: In-Reply-To:To:Cc; b=POPEYJMVg2K9nqoyYVtjuL+yMF9QvBplzgBZm52xIWbGayTV8Muk5pGRu4khaGz0Bt9r9rcqmaVxRQb4DaEHiU3qo25JuItZs4y91gw6wyr8GFLMk8pErQBkSyQz3VXalq3A4q2JPEz2r1xx6dnePOMZqJBDiVoNEELrbCPk9VQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=iJJXpIHC; arc=none smtp.client-ip=209.85.216.50 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="iJJXpIHC" Received: by mail-pj1-f50.google.com with SMTP id 98e67ed59e1d1-311e46d38ddso2890512a91.0; Sat, 12 Jul 2025 14:32:10 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1752355930; x=1752960730; darn=vger.kernel.org; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:from:to:cc:subject:date:message-id :reply-to; bh=HtRQST6oRjj0LgYjY2mtVgF/hjywiLbC7jwSxYoLgQA=; b=iJJXpIHC1G9ehA6QxYVCLu+GdWjiEm848O7eXnlgoFg5e8uqrANwyi7yZVb0bkTT/n Y1DgK21+BLZXF80o6+a4nZkMhUrhnefFyAjo1IED8Ri+GXVI6foPdo4kKM5PwVBTLYnz fyCx6sanjb+19aO8RKDo4l7L+E9L7QUsvHHV0Ve7zUlZebMu8zyNY5ZP25YXBpwCrrxP 0EL0xzc7sGDsVSY+esvGjrK2ZI5n6O/7588UnzdIUkzkTPZZo8NdwXNUas+gE9MuLimR Azf79HwD5SJCLqKzhu40DOcX3byTTwLKWxmp7H1/yagcBJk4JzTs7IVkEQF438yVexX1 JO0g== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752355930; x=1752960730; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=HtRQST6oRjj0LgYjY2mtVgF/hjywiLbC7jwSxYoLgQA=; b=f3GgA8YYjDl+8q9PFcd0f/MVaC4qvVIiZi9t4XYYc2d5tmZU4/T7XVbp63tygMBXny /ni6OMaVCqYWxsUvbw2pVdFHOe239QovzYjZihEPvBHWmpbJrE9NM8rnTX+oM5xRv4Yf jjPGfE+9P6KdUO/bjIpZ+AlMqEMFAJwKpnQLsw3mnJfhR/2lR78zGKCV8gzw/nzvmIPS xuUcK3AjCaHONGZH0zqwzDAMXviw+GSO+NztXihm95CxdDRg4IqtsZvyun8J4s6aD0+K wibDyl8HtCirq+le89aNc9UNadudRreiQ15Gznfhfz9HHmGn00rdQwXpz6dSaBLMDZXA MogQ== X-Forwarded-Encrypted: i=1; AJvYcCUgSyd4bRh56TvMpd/20E+iV2e/9nvUGXuPtTBla+nCrdshwxmZvs2ayfcmDpek0LtdstypT8Gf+6JM3uKwBg==@vger.kernel.org X-Gm-Message-State: AOJu0Yz85E5Wt9psBwYTmFrpg5WG1ODokZ108gC8kZlmb67Z4Rd6h+b9 IozcdNYKgM6dZYy71MLC/QGwtpXojygDROrFNCt44twxMOeiJtYZ4112 X-Gm-Gg: ASbGncvYU0xS6GjZe5MQQVJNzcvrFquCgi2XsRUaKCxHa3PKxMckviiF79w+o6bb63n KWhx86YK230knZhaOF3yFV0oxjh4OXtmx7ITRFF5JGpm94HjK4YSduLv/FIupvwc2qTdx0yitkE Q1+eZDwFGyJsCXlGZggEneBP6J/bni8tR+18TT+2+gMUAnq/Txlbu9ukdDha8G5uGfjaKZ0aY5l TwMZqQQFNScZobTZwQdEBZe1Bj2kR56nhSHsPoa2D1I/F2//O3Agx3jE9TB0iLFIJXKfrAx9YSg Jdanr4cFqF43Hul3I5v20cStYiDaTXFIK9Bwslb7ZywKm8xdc6eShTYnpi5bpX0PhTsp4hGwsWv ccZOawXrPpopi7oZo2Vy2uVcjbCr74gHlPRyRJukoumM/5A== X-Google-Smtp-Source: AGHT+IH4o4Ul3dJAPkL8Zf1bVxHe/F3eh4QfvlGGzhfBDOyFuZ6X4iobwmvAQbZVaFZoUk34LWX8BA== X-Received: by 2002:a17:90b:2f03:b0:312:e90b:419e with SMTP id 98e67ed59e1d1-31c50d7b487mr11598155a91.12.1752355929612; Sat, 12 Jul 2025 14:32:09 -0700 (PDT) Received: from mitchelllevy.localdomain ([174.127.224.194]) by smtp.gmail.com with ESMTPSA id 98e67ed59e1d1-31c3eb65aa2sm8003935a91.40.2025.07.12.14.32.08 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 12 Jul 2025 14:32:09 -0700 (PDT) From: Mitchell Levy Date: Sat, 12 Jul 2025 14:31:16 -0700 Subject: [PATCH v2 5/5] rust: percpu: cache per-CPU pointers in the dynamic case Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Message-Id: <20250712-rust-percpu-v2-5-826f2567521b@gmail.com> References: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> In-Reply-To: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> To: Miguel Ojeda , Alex Gaynor , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Andreas Hindborg , Alice Ryhl , Trevor Gross , Andrew Morton , Dennis Zhou , Tejun Heo , Christoph Lameter , Danilo Krummrich , Benno Lossin Cc: linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-mm@kvack.org, Mitchell Levy X-Mailer: b4 0.14.0 X-Developer-Signature: v=1; a=ed25519-sha256; t=1752355923; l=3148; i=levymitchell0@gmail.com; s=20240719; h=from:subject:message-id; bh=e1kjOZMlHir0QAwDPK2Mh6knOwIrBRKVgMEnFQ0sips=; b=SNmvATz+HCHFQwUB26nikspj2mxM7yzHBgPaqP8iLy3oOOIsHr2Ff3x4m6cZ0iwMHZviz/pI0 7ZWaguSukFNCvS4HxAy06fKXvmysfDraT3Dh5NuKZoVezP+qab4M0nC X-Developer-Key: i=levymitchell0@gmail.com; a=ed25519; pk=n6kBmUnb+UNmjVkTnDwrLwTJAEKUfs2e8E+MFPZI93E= Currently, the creation of a `PerCpuNumeric` requires a memory read via the `Arc` managing the dynamic allocation. While the compiler might be clever enough to consolidate these reads in some cases, the read must happen *somewhere*, which, when we're concerning ourselves with individual instructions, is a very high burden. Instead, cache the `PerCpuPointer` inside the `DynamicPerCpu` structure; then, the `Arc` is used solely to manage the allocation. Signed-off-by: Mitchell Levy --- rust/kernel/percpu.rs | 12 +++++++++--- rust/kernel/percpu/numeric.rs | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs index b97d1d07a614..7458fe413f25 100644 --- a/rust/kernel/percpu.rs +++ b/rust/kernel/percpu.rs @@ -26,7 +26,10 @@ =20 /// Holds a dynamically-allocated per-CPU variable. pub struct DynamicPerCpu { + // INVARIANT: `ptr` is managed by `alloc` and the value of `ptr` does = not change for the + // lifetime of `self`. alloc: Arc>, + ptr: PerCpuPtr, } =20 /// Holds a statically-allocated per-CPU variable. @@ -204,9 +207,10 @@ impl DynamicPerCpu { pub fn new(flags: Flags) -> Option { let alloc: PerCpuAllocation =3D PerCpuAllocation::new()?; =20 + let ptr =3D alloc.0; let arc =3D Arc::new(alloc, flags).ok()?; =20 - Some(Self { alloc: arc }) + Some(Self { alloc: arc, ptr }) } } =20 @@ -217,8 +221,9 @@ impl DynamicPerCpu { /// * `alloc` - The allocation to use /// * `flags` - The flags used to allocate an `Arc` that keeps track o= f the `PerCpuAllocation`. pub fn new_from_allocation(alloc: PerCpuAllocation, flags: Flags) -= > Option { + let ptr =3D alloc.0; let arc =3D Arc::new(alloc, flags).ok()?; - Some(Self { alloc: arc }) + Some(Self { alloc: arc, ptr }) } } =20 @@ -226,7 +231,7 @@ pub fn new_from_allocation(alloc: PerCpuAllocation, = flags: Flags) -> Option PerCpu for DynamicPerCpu { unsafe fn ptr(&mut self) -> &PerCpuPtr { - &self.alloc.0 + &self.ptr } } =20 @@ -234,6 +239,7 @@ impl Clone for DynamicPerCpu { fn clone(&self) -> Self { Self { alloc: self.alloc.clone(), + ptr: self.ptr, } } } diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs index e4008f872af1..1b37cc7e5c19 100644 --- a/rust/kernel/percpu/numeric.rs +++ b/rust/kernel/percpu/numeric.rs @@ -62,7 +62,7 @@ impl DynamicPerCpu<$ty> { /// Returns a `PerCpuNumeric` that can be used to manipulate t= he underlying per-CPU /// variable. pub fn num(&self) -> PerCpuNumeric<'_, $ty> { - PerCpuNumeric { ptr: &self.alloc.0 } + PerCpuNumeric { ptr: &self.ptr } } } impl StaticPerCpu<$ty> { --=20 2.34.1