From: Boqun Feng <boqun.feng@gmail.com>
In order to synchronize with C or external memory, atomic operations
over raw pointers are need. Although there is already an
`Atomic::from_ptr()` to provide a `&Atomic<T>`, it's more convenient to
have helpers that directly perform atomic operations on raw pointers.
Hence a few are added, which are basically an `Atomic::from_ptr().op()`
wrapper.
Note: for naming, since `atomic_xchg()` and `atomic_cmpxchg()` have a
conflict naming to 32bit C atomic xchg/cmpxchg, hence the helpers are
just named as `xchg()` and `cmpxchg()`. For `atomic_load()` and
`atomic_store()`, their 32bit C counterparts are `atomic_read()` and
`atomic_set()`, so keep the `atomic_` prefix.
[boqun: Fix typo spotted by Alice and fix broken sentence spotted by
Gary]
Reviewed-by: Alice Ryhl <aliceryhl@google.com>
Reviewed-by: Gary Guo <gary@garyguo.net>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
Link: https://patch.msgid.link/20260120115207.55318-3-boqun.feng@gmail.com
---
rust/kernel/sync/atomic.rs | 104 +++++++++++++++++++++++++++
rust/kernel/sync/atomic/predefine.rs | 46 ++++++++++++
2 files changed, 150 insertions(+)
diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs
index f80cebce5bc1..1bb1fc2be177 100644
--- a/rust/kernel/sync/atomic.rs
+++ b/rust/kernel/sync/atomic.rs
@@ -703,3 +703,107 @@ pub fn cmpxchg<Ordering: ordering::Ordering>(
}
}
}
+
+/// Atomic load over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work
+/// with C side on synchronizations:
+///
+/// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication.
+/// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent store from kernel (C or Rust), it has to be atomic.
+#[doc(alias("READ_ONCE", "smp_load_acquire"))]
+#[inline(always)]
+pub unsafe fn atomic_load<T: AtomicType, Ordering: ordering::AcquireOrRelaxed>(
+ ptr: *mut T,
+ o: Ordering,
+) -> T
+where
+ T::Repr: AtomicBasicOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent stores from kernel are atomic, hence no data race per
+ // LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.load(o)
+}
+
+/// Atomic store over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work
+/// with C side on synchronizations:
+///
+/// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication.
+/// - `atomic_load(.., Release)` maps to `smp_store_release()`.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[doc(alias("WRITE_ONCE", "smp_store_release"))]
+#[inline(always)]
+pub unsafe fn atomic_store<T: AtomicType, Ordering: ordering::ReleaseOrRelaxed>(
+ ptr: *mut T,
+ v: T,
+ o: Ordering,
+) where
+ T::Repr: AtomicBasicOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.store(v, o);
+}
+
+/// Atomic exchange over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work
+/// with C side on synchronizations.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[inline(always)]
+pub unsafe fn xchg<T: AtomicType, Ordering: ordering::Ordering>(
+ ptr: *mut T,
+ new: T,
+ o: Ordering,
+) -> T
+where
+ T::Repr: AtomicExchangeOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.xchg(new, o)
+}
+
+/// Atomic compare and exchange over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work
+/// with C side on synchronizations.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[doc(alias("try_cmpxchg"))]
+#[inline(always)]
+pub unsafe fn cmpxchg<T: AtomicType, Ordering: ordering::Ordering>(
+ ptr: *mut T,
+ old: T,
+ new: T,
+ o: Ordering,
+) -> Result<T, T>
+where
+ T::Repr: AtomicExchangeOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o)
+}
diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs
index ceb3caed9784..1d53834fcb12 100644
--- a/rust/kernel/sync/atomic/predefine.rs
+++ b/rust/kernel/sync/atomic/predefine.rs
@@ -178,6 +178,14 @@ fn atomic_basic_tests() {
assert_eq!(v, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) });
+ });
}
#[test]
@@ -188,6 +196,17 @@ fn atomic_acquire_release_tests() {
x.store(v, Release);
assert_eq!(v, x.load(Acquire));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(0);
+ let ptr = x.as_ptr();
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ unsafe { atomic_store(ptr, v, Release) };
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(v, unsafe { atomic_load(ptr, Acquire) });
+ });
}
#[test]
@@ -201,6 +220,18 @@ fn atomic_xchg_tests() {
assert_eq!(old, x.xchg(new, Full));
assert_eq!(new, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ let old = v;
+ let new = v + 1;
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(old, unsafe { xchg(ptr, new, Full) });
+ assert_eq!(new, x.load(Relaxed));
+ });
}
#[test]
@@ -216,6 +247,21 @@ fn atomic_cmpxchg_tests() {
assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed));
assert_eq!(new, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ let old = v;
+ let new = v + 1;
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) });
+ assert_eq!(old, x.load(Relaxed));
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) });
+ assert_eq!(new, x.load(Relaxed));
+ });
}
#[test]
--
2.50.1 (Apple Git-155)The following commit has been merged into the locking/core branch of tip:
Commit-ID: e2f9c86f33abb89d3e52436018f58e5fb951cc04
Gitweb: https://git.kernel.org/tip/e2f9c86f33abb89d3e52436018f58e5fb951cc04
Author: Boqun Feng <boqun.feng@gmail.com>
AuthorDate: Tue, 03 Mar 2026 12:16:58 -08:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sun, 08 Mar 2026 11:06:50 +01:00
rust: sync: atomic: Add atomic operation helpers over raw pointers
In order to synchronize with C or external memory, atomic operations
over raw pointers are need. Although there is already an
`Atomic::from_ptr()` to provide a `&Atomic<T>`, it's more convenient to
have helpers that directly perform atomic operations on raw pointers.
Hence a few are added, which are basically an `Atomic::from_ptr().op()`
wrapper.
Note: for naming, since `atomic_xchg()` and `atomic_cmpxchg()` have a
conflict naming to 32bit C atomic xchg/cmpxchg, hence the helpers are
just named as `xchg()` and `cmpxchg()`. For `atomic_load()` and
`atomic_store()`, their 32bit C counterparts are `atomic_read()` and
`atomic_set()`, so keep the `atomic_` prefix.
[boqun: Fix typo spotted by Alice and fix broken sentence spotted by
Gary]
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Alice Ryhl <aliceryhl@google.com>
Reviewed-by: Gary Guo <gary@garyguo.net>
Link: https://patch.msgid.link/20260120115207.55318-3-boqun.feng@gmail.com
Link: https://patch.msgid.link/20260303201701.12204-11-boqun@kernel.org
---
rust/kernel/sync/atomic.rs | 104 ++++++++++++++++++++++++++-
rust/kernel/sync/atomic/predefine.rs | 46 ++++++++++++-
2 files changed, 150 insertions(+)
diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs
index f80cebc..1bb1fc2 100644
--- a/rust/kernel/sync/atomic.rs
+++ b/rust/kernel/sync/atomic.rs
@@ -703,3 +703,107 @@ impl AtomicFlag {
}
}
}
+
+/// Atomic load over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work
+/// with C side on synchronizations:
+///
+/// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication.
+/// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent store from kernel (C or Rust), it has to be atomic.
+#[doc(alias("READ_ONCE", "smp_load_acquire"))]
+#[inline(always)]
+pub unsafe fn atomic_load<T: AtomicType, Ordering: ordering::AcquireOrRelaxed>(
+ ptr: *mut T,
+ o: Ordering,
+) -> T
+where
+ T::Repr: AtomicBasicOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent stores from kernel are atomic, hence no data race per
+ // LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.load(o)
+}
+
+/// Atomic store over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work
+/// with C side on synchronizations:
+///
+/// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication.
+/// - `atomic_load(.., Release)` maps to `smp_store_release()`.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[doc(alias("WRITE_ONCE", "smp_store_release"))]
+#[inline(always)]
+pub unsafe fn atomic_store<T: AtomicType, Ordering: ordering::ReleaseOrRelaxed>(
+ ptr: *mut T,
+ v: T,
+ o: Ordering,
+) where
+ T::Repr: AtomicBasicOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.store(v, o);
+}
+
+/// Atomic exchange over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work
+/// with C side on synchronizations.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[inline(always)]
+pub unsafe fn xchg<T: AtomicType, Ordering: ordering::Ordering>(
+ ptr: *mut T,
+ new: T,
+ o: Ordering,
+) -> T
+where
+ T::Repr: AtomicExchangeOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.xchg(new, o)
+}
+
+/// Atomic compare and exchange over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work
+/// with C side on synchronizations.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[doc(alias("try_cmpxchg"))]
+#[inline(always)]
+pub unsafe fn cmpxchg<T: AtomicType, Ordering: ordering::Ordering>(
+ ptr: *mut T,
+ old: T,
+ new: T,
+ o: Ordering,
+) -> Result<T, T>
+where
+ T::Repr: AtomicExchangeOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o)
+}
diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs
index ceb3cae..1d53834 100644
--- a/rust/kernel/sync/atomic/predefine.rs
+++ b/rust/kernel/sync/atomic/predefine.rs
@@ -178,6 +178,14 @@ mod tests {
assert_eq!(v, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) });
+ });
}
#[test]
@@ -188,6 +196,17 @@ mod tests {
x.store(v, Release);
assert_eq!(v, x.load(Acquire));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(0);
+ let ptr = x.as_ptr();
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ unsafe { atomic_store(ptr, v, Release) };
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(v, unsafe { atomic_load(ptr, Acquire) });
+ });
}
#[test]
@@ -201,6 +220,18 @@ mod tests {
assert_eq!(old, x.xchg(new, Full));
assert_eq!(new, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ let old = v;
+ let new = v + 1;
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(old, unsafe { xchg(ptr, new, Full) });
+ assert_eq!(new, x.load(Relaxed));
+ });
}
#[test]
@@ -216,6 +247,21 @@ mod tests {
assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed));
assert_eq!(new, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ let old = v;
+ let new = v + 1;
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) });
+ assert_eq!(old, x.load(Relaxed));
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) });
+ assert_eq!(new, x.load(Relaxed));
+ });
}
#[test]© 2016 - 2026 Red Hat, Inc.