tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

rust: sync: atomic: Add atomic operation helpers over raw pointers

In order to synchronize with C or external memory, atomic operations
over raw pointers are need. Although there is already an
`Atomic::from_ptr()` to provide a `&Atomic<T>`, it's more convenient to
have helpers that directly perform atomic operations on raw pointers.
Hence a few are added, which are basically an `Atomic::from_ptr().op()`
wrapper.

Note: for naming, since `atomic_xchg()` and `atomic_cmpxchg()` have a
conflict naming to 32bit C atomic xchg/cmpxchg, hence the helpers are
just named as `xchg()` and `cmpxchg()`. For `atomic_load()` and
`atomic_store()`, their 32bit C counterparts are `atomic_read()` and
`atomic_set()`, so keep the `atomic_` prefix.

[boqun: Fix typo spotted by Alice and fix broken sentence spotted by
Gary]

Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Alice Ryhl <aliceryhl@google.com>
Reviewed-by: Gary Guo <gary@garyguo.net>
Link: https://patch.msgid.link/20260120115207.55318-3-boqun.feng@gmail.com
Link: https://patch.msgid.link/20260303201701.12204-11-boqun@kernel.org

authored by

Boqun Feng and committed by
Peter Zijlstra e2f9c86f 28286620

+150
+104
rust/kernel/sync/atomic.rs
··· 703 703 } 704 704 } 705 705 } 706 + 707 + /// Atomic load over raw pointers. 708 + /// 709 + /// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work 710 + /// with C side on synchronizations: 711 + /// 712 + /// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication. 713 + /// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`. 714 + /// 715 + /// # Safety 716 + /// 717 + /// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. 718 + /// - If there is a concurrent store from kernel (C or Rust), it has to be atomic. 719 + #[doc(alias("READ_ONCE", "smp_load_acquire"))] 720 + #[inline(always)] 721 + pub unsafe fn atomic_load<T: AtomicType, Ordering: ordering::AcquireOrRelaxed>( 722 + ptr: *mut T, 723 + o: Ordering, 724 + ) -> T 725 + where 726 + T::Repr: AtomicBasicOps, 727 + { 728 + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to 729 + // `align_of::<T>()`, and all concurrent stores from kernel are atomic, hence no data race per 730 + // LKMM. 731 + unsafe { Atomic::from_ptr(ptr) }.load(o) 732 + } 733 + 734 + /// Atomic store over raw pointers. 735 + /// 736 + /// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work 737 + /// with C side on synchronizations: 738 + /// 739 + /// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication. 740 + /// - `atomic_load(.., Release)` maps to `smp_store_release()`. 741 + /// 742 + /// # Safety 743 + /// 744 + /// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. 745 + /// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. 746 + #[doc(alias("WRITE_ONCE", "smp_store_release"))] 747 + #[inline(always)] 748 + pub unsafe fn atomic_store<T: AtomicType, Ordering: ordering::ReleaseOrRelaxed>( 749 + ptr: *mut T, 750 + v: T, 751 + o: Ordering, 752 + ) where 753 + T::Repr: AtomicBasicOps, 754 + { 755 + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to 756 + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race 757 + // per LKMM. 758 + unsafe { Atomic::from_ptr(ptr) }.store(v, o); 759 + } 760 + 761 + /// Atomic exchange over raw pointers. 762 + /// 763 + /// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work 764 + /// with C side on synchronizations. 765 + /// 766 + /// # Safety 767 + /// 768 + /// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. 769 + /// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. 770 + #[inline(always)] 771 + pub unsafe fn xchg<T: AtomicType, Ordering: ordering::Ordering>( 772 + ptr: *mut T, 773 + new: T, 774 + o: Ordering, 775 + ) -> T 776 + where 777 + T::Repr: AtomicExchangeOps, 778 + { 779 + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to 780 + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race 781 + // per LKMM. 782 + unsafe { Atomic::from_ptr(ptr) }.xchg(new, o) 783 + } 784 + 785 + /// Atomic compare and exchange over raw pointers. 786 + /// 787 + /// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work 788 + /// with C side on synchronizations. 789 + /// 790 + /// # Safety 791 + /// 792 + /// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. 793 + /// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. 794 + #[doc(alias("try_cmpxchg"))] 795 + #[inline(always)] 796 + pub unsafe fn cmpxchg<T: AtomicType, Ordering: ordering::Ordering>( 797 + ptr: *mut T, 798 + old: T, 799 + new: T, 800 + o: Ordering, 801 + ) -> Result<T, T> 802 + where 803 + T::Repr: AtomicExchangeOps, 804 + { 805 + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to 806 + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race 807 + // per LKMM. 808 + unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o) 809 + }
+46
rust/kernel/sync/atomic/predefine.rs
··· 178 178 179 179 assert_eq!(v, x.load(Relaxed)); 180 180 }); 181 + 182 + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 183 + let x = Atomic::new(v); 184 + let ptr = x.as_ptr(); 185 + 186 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 187 + assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) }); 188 + }); 181 189 } 182 190 183 191 #[test] ··· 195 187 196 188 x.store(v, Release); 197 189 assert_eq!(v, x.load(Acquire)); 190 + }); 191 + 192 + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 193 + let x = Atomic::new(0); 194 + let ptr = x.as_ptr(); 195 + 196 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 197 + unsafe { atomic_store(ptr, v, Release) }; 198 + 199 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 200 + assert_eq!(v, unsafe { atomic_load(ptr, Acquire) }); 198 201 }); 199 202 } 200 203 ··· 218 199 let new = v + 1; 219 200 220 201 assert_eq!(old, x.xchg(new, Full)); 202 + assert_eq!(new, x.load(Relaxed)); 203 + }); 204 + 205 + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 206 + let x = Atomic::new(v); 207 + let ptr = x.as_ptr(); 208 + 209 + let old = v; 210 + let new = v + 1; 211 + 212 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 213 + assert_eq!(old, unsafe { xchg(ptr, new, Full) }); 221 214 assert_eq!(new, x.load(Relaxed)); 222 215 }); 223 216 } ··· 245 214 assert_eq!(Err(old), x.cmpxchg(new, new, Full)); 246 215 assert_eq!(old, x.load(Relaxed)); 247 216 assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed)); 217 + assert_eq!(new, x.load(Relaxed)); 218 + }); 219 + 220 + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 221 + let x = Atomic::new(v); 222 + let ptr = x.as_ptr(); 223 + 224 + let old = v; 225 + let new = v + 1; 226 + 227 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 228 + assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) }); 229 + assert_eq!(old, x.load(Relaxed)); 230 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 231 + assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) }); 248 232 assert_eq!(new, x.load(Relaxed)); 249 233 }); 250 234 }