rust: sync: atomic: Add performance-optimal Flag type for atomic booleans

Add AtomicFlag type for boolean flags.

Document when AtomicFlag is generally preferable to Atomic<bool>: in
particular, when RMW operations such as xchg()/cmpxchg() may be used
and minimizing memory usage is not the top priority. On some
architectures without byte-sized RMW instructions, Atomic<bool> can be
slower for RMW operations.

Signed-off-by: FUJITA Tomonori <fujita.tomonori@gmail.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Gary Guo <gary@garyguo.net>
Link: https://patch.msgid.link/20260129122622.3896144-2-tomo@aliasing.net
Link: https://patch.msgid.link/20260303201701.12204-9-boqun@kernel.org

authored by

FUJITA Tomonori and committed by

Peter Zijlstra 3 months ago ec6fc66a ac8f06ad

+142

2 changed files

expand all

rust

kernel

sync

atomic

predefine.rs

atomic.rs

+125

rust/kernel/sync/atomic.rs

··· 578 578 unsafe { from_repr(ret) } 579 579 } 580 580 } 581 + 582 + #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] 583 + #[repr(C)] 584 + #[derive(Clone, Copy)] 585 + struct Flag { 586 + bool_field: bool, 587 + } 588 + 589 + /// # Invariants 590 + /// 591 + /// `padding` must be all zeroes. 592 + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 593 + #[repr(C, align(4))] 594 + #[derive(Clone, Copy)] 595 + struct Flag { 596 + #[cfg(target_endian = "big")] 597 + padding: [u8; 3], 598 + bool_field: bool, 599 + #[cfg(target_endian = "little")] 600 + padding: [u8; 3], 601 + } 602 + 603 + impl Flag { 604 + #[inline(always)] 605 + const fn new(b: bool) -> Self { 606 + // INVARIANT: `padding` is all zeroes. 607 + Self { 608 + bool_field: b, 609 + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 610 + padding: [0; 3], 611 + } 612 + } 613 + } 614 + 615 + // SAFETY: `Flag` and `Repr` have the same size and alignment, and `Flag` is round-trip 616 + // transmutable to the selected representation (`i8` or `i32`). 617 + unsafe impl AtomicType for Flag { 618 + #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] 619 + type Repr = i8; 620 + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 621 + type Repr = i32; 622 + } 623 + 624 + /// An atomic flag type intended to be backed by performance-optimal integer type. 625 + /// 626 + /// The backing integer type is an implementation detail; it may vary by architecture and change 627 + /// in the future. 628 + /// 629 + /// [`AtomicFlag`] is generally preferable to [`Atomic<bool>`] when you need read-modify-write 630 + /// (RMW) operations (e.g. [`Atomic::xchg()`]/[`Atomic::cmpxchg()`]) or when [`Atomic<bool>`] does 631 + /// not save memory due to padding. On some architectures that do not support byte-sized atomic 632 + /// RMW operations, RMW operations on [`Atomic<bool>`] are slower. 633 + /// 634 + /// If you only use [`Atomic::load()`]/[`Atomic::store()`], [`Atomic<bool>`] is fine. 635 + /// 636 + /// # Examples 637 + /// 638 + /// ``` 639 + /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; 640 + /// 641 + /// let flag = AtomicFlag::new(false); 642 + /// assert_eq!(false, flag.load(Relaxed)); 643 + /// flag.store(true, Relaxed); 644 + /// assert_eq!(true, flag.load(Relaxed)); 645 + /// ``` 646 + pub struct AtomicFlag(Atomic<Flag>); 647 + 648 + impl AtomicFlag { 649 + /// Creates a new atomic flag. 650 + #[inline(always)] 651 + pub const fn new(b: bool) -> Self { 652 + Self(Atomic::new(Flag::new(b))) 653 + } 654 + 655 + /// Returns a mutable reference to the underlying flag as a [`bool`]. 656 + /// 657 + /// This is safe because the mutable reference of the atomic flag guarantees exclusive access. 658 + /// 659 + /// # Examples 660 + /// 661 + /// ``` 662 + /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; 663 + /// 664 + /// let mut atomic_flag = AtomicFlag::new(false); 665 + /// assert_eq!(false, atomic_flag.load(Relaxed)); 666 + /// *atomic_flag.get_mut() = true; 667 + /// assert_eq!(true, atomic_flag.load(Relaxed)); 668 + /// ``` 669 + #[inline(always)] 670 + pub fn get_mut(&mut self) -> &mut bool { 671 + &mut self.0.get_mut().bool_field 672 + } 673 + 674 + /// Loads the value from the atomic flag. 675 + #[inline(always)] 676 + pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, o: Ordering) -> bool { 677 + self.0.load(o).bool_field 678 + } 679 + 680 + /// Stores a value to the atomic flag. 681 + #[inline(always)] 682 + pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: bool, o: Ordering) { 683 + self.0.store(Flag::new(v), o); 684 + } 685 + 686 + /// Stores a value to the atomic flag and returns the previous value. 687 + #[inline(always)] 688 + pub fn xchg<Ordering: ordering::Ordering>(&self, new: bool, o: Ordering) -> bool { 689 + self.0.xchg(Flag::new(new), o).bool_field 690 + } 691 + 692 + /// Store a value to the atomic flag if the current value is equal to `old`. 693 + #[inline(always)] 694 + pub fn cmpxchg<Ordering: ordering::Ordering>( 695 + &self, 696 + old: bool, 697 + new: bool, 698 + o: Ordering, 699 + ) -> Result<bool, bool> { 700 + match self.0.cmpxchg(Flag::new(old), Flag::new(new), o) { 701 + Ok(_) => Ok(old), 702 + Err(f) => Err(f.bool_field), 703 + } 704 + } 705 + }

+17

rust/kernel/sync/atomic/predefine.rs

··· 272 272 ); 273 273 assert_eq!(x.load(Relaxed), &raw const u); 274 274 } 275 + 276 + #[test] 277 + fn atomic_flag_tests() { 278 + let mut flag = AtomicFlag::new(false); 279 + 280 + assert_eq!(false, flag.load(Relaxed)); 281 + 282 + *flag.get_mut() = true; 283 + assert_eq!(true, flag.load(Relaxed)); 284 + 285 + assert_eq!(true, flag.xchg(false, Relaxed)); 286 + assert_eq!(false, flag.load(Relaxed)); 287 + 288 + *flag.get_mut() = true; 289 + assert_eq!(Ok(true), flag.cmpxchg(true, false, Full)); 290 + assert_eq!(false, flag.load(Relaxed)); 291 + } 275 292 }

Configure Feed

Configure Feed