Currently, the creation of a `PerCpuNumeric` requires a memory read via
the `Arc` managing the dynamic allocation. While the compiler might be
clever enough to consolidate these reads in some cases, the read must
happen *somewhere*, which, when we're concerning ourselves with
individual instructions, is a very high burden.
Instead, cache the `PerCpuPointer` inside the `DynamicPerCpu` structure;
then, the `Arc` is used solely to manage the allocation.
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/kernel/percpu/dynamic.rs | 8 ++++++--
rust/kernel/percpu/numeric.rs | 4 ++--
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
index aad08e4b4251..8683e94bcd9f 100644
--- a/rust/kernel/percpu/dynamic.rs
+++ b/rust/kernel/percpu/dynamic.rs
@@ -58,7 +58,10 @@ fn drop(&mut self) {
pub struct DynamicPerCpu<T> {
// INVARIANT: The memory location in each CPU's per-CPU area pointed at by `alloc.0` has been
// initialized.
+ // INVARIANT: `ptr` is the per-CPU pointer managed by `alloc`, which does not change for the
+ // lifetime of `self`.
pub(super) alloc: Arc<PerCpuAllocation<T>>,
+ pub(super) ptr: PerCpuPtr<T>,
}
impl<T: Zeroable> DynamicPerCpu<T> {
@@ -70,9 +73,10 @@ impl<T: Zeroable> DynamicPerCpu<T> {
pub fn new_zero(flags: Flags) -> Option<Self> {
let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_zero()?;
+ let ptr = alloc.0;
let arc = Arc::new(alloc, flags).ok()?;
- Some(Self { alloc: arc })
+ Some(Self { alloc: arc, ptr })
}
}
@@ -102,7 +106,7 @@ pub fn new_with(val: T, flags: Flags) -> Option<Self> {
let arc = Arc::new(alloc, flags).ok()?;
- Some(Self { alloc: arc })
+ Some(Self { alloc: arc, ptr })
}
}
diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs
index 4de93f653f0e..3412c2de85fd 100644
--- a/rust/kernel/percpu/numeric.rs
+++ b/rust/kernel/percpu/numeric.rs
@@ -20,7 +20,7 @@ impl DynamicPerCpu<$ty> {
pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
// The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
// this pointer is valid and initialized on all CPUs.
- PerCpuNumeric { ptr: &self.alloc.0 }
+ PerCpuNumeric { ptr: &self.ptr }
}
}
impl StaticPerCpu<$ty> {
@@ -71,7 +71,7 @@ impl DynamicPerCpu<$ty> {
pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
// The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
// this pointer is valid and initialized on all CPUs.
- PerCpuNumeric { ptr: &self.alloc.0 }
+ PerCpuNumeric { ptr: &self.ptr }
}
}
impl StaticPerCpu<$ty> {
--
2.34.1