Merge tag 'rust-6.4' of https://github.com/Rust-for-Linux/linux

+1

MAINTAINERS

··· 18346 18346 R: Boqun Feng <boqun.feng@gmail.com> 18347 18347 R: Gary Guo <gary@garyguo.net> 18348 18348 R: Björn Roy Baron <bjorn3_gh@protonmail.com> 18349 + R: Benno Lossin <benno.lossin@proton.me> 18349 18350 L: rust-for-linux@vger.kernel.org 18350 18351 S: Supported 18351 18352 W: https://github.com/Rust-for-Linux/linux

+1

rust/.gitignore

··· 2 2 3 3 bindings_generated.rs 4 4 bindings_helpers_generated.rs 5 + uapi_generated.rs 5 6 exports_*_generated.h 6 7 doc/ 7 8 test/

+23 -5

rust/Makefile

··· 16 16 always-$(CONFIG_RUST) += exports_alloc_generated.h exports_bindings_generated.h \ 17 17 exports_kernel_generated.h 18 18 19 + always-$(CONFIG_RUST) += uapi/uapi_generated.rs 20 + obj-$(CONFIG_RUST) += uapi.o 21 + 19 22 ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW 20 23 obj-$(CONFIG_RUST) += build_error.o 21 24 else ··· 116 113 117 114 rustdoc-kernel: private rustc_target_flags = --extern alloc \ 118 115 --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \ 119 - --extern bindings 116 + --extern bindings --extern uapi 120 117 rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \ 121 118 rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \ 122 119 $(obj)/bindings.o FORCE ··· 142 139 $(call if_changed,rustc_test_library) 143 140 144 141 rusttestlib-bindings: $(src)/bindings/lib.rs rusttest-prepare FORCE 142 + $(call if_changed,rustc_test_library) 143 + 144 + rusttestlib-uapi: $(src)/uapi/lib.rs rusttest-prepare FORCE 145 145 $(call if_changed,rustc_test_library) 146 146 147 147 quiet_cmd_rustdoc_test = RUSTDOC T $< ··· 229 223 $(call if_changed,rustdoc_test) 230 224 231 225 rusttest-kernel: private rustc_target_flags = --extern alloc \ 232 - --extern build_error --extern macros --extern bindings 226 + --extern build_error --extern macros --extern bindings --extern uapi 233 227 rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \ 234 - rusttestlib-build_error rusttestlib-macros rusttestlib-bindings FORCE 228 + rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \ 229 + rusttestlib-uapi FORCE 235 230 $(call if_changed,rustc_test) 236 231 $(call if_changed,rustc_test_library) 237 232 ··· 306 299 $(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \ 307 300 $(shell grep -v '^#\|^$$' $(srctree)/$(src)/bindgen_parameters) 308 301 $(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \ 302 + $(src)/bindgen_parameters FORCE 303 + $(call if_changed_dep,bindgen) 304 + 305 + $(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \ 306 + $(shell grep -v '^#\|^$$' $(srctree)/$(src)/bindgen_parameters) 307 + $(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \ 309 308 $(src)/bindgen_parameters FORCE 310 309 $(call if_changed_dep,bindgen) 311 310 ··· 415 402 $(obj)/bindings/bindings_helpers_generated.rs FORCE 416 403 $(call if_changed_dep,rustc_library) 417 404 405 + $(obj)/uapi.o: $(src)/uapi/lib.rs \ 406 + $(obj)/compiler_builtins.o \ 407 + $(obj)/uapi/uapi_generated.rs FORCE 408 + $(call if_changed_dep,rustc_library) 409 + 418 410 $(obj)/kernel.o: private rustc_target_flags = --extern alloc \ 419 - --extern build_error --extern macros --extern bindings 411 + --extern build_error --extern macros --extern bindings --extern uapi 420 412 $(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \ 421 - $(obj)/libmacros.so $(obj)/bindings.o FORCE 413 + $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE 422 414 $(call if_changed_dep,rustc_library) 423 415 424 416 endif # CONFIG_RUST

+134 -3

rust/alloc/vec/mod.rs

··· 122 122 #[cfg(not(no_global_oom_handling))] 123 123 mod spec_from_elem; 124 124 125 - #[cfg(not(no_global_oom_handling))] 126 125 use self::set_len_on_drop::SetLenOnDrop; 127 126 128 - #[cfg(not(no_global_oom_handling))] 129 127 mod set_len_on_drop; 130 128 131 129 #[cfg(not(no_global_oom_handling))] ··· 147 149 #[cfg(not(no_global_oom_handling))] 148 150 use self::spec_extend::SpecExtend; 149 151 150 - #[cfg(not(no_global_oom_handling))] 152 + use self::spec_extend::TrySpecExtend; 153 + 151 154 mod spec_extend; 152 155 153 156 /// A contiguous growable array type, written as `Vec<T>`, short for 'vector'. ··· 1918 1919 self.len += count; 1919 1920 } 1920 1921 1922 + /// Tries to append elements to `self` from other buffer. 1923 + #[inline] 1924 + unsafe fn try_append_elements(&mut self, other: *const [T]) -> Result<(), TryReserveError> { 1925 + let count = unsafe { (*other).len() }; 1926 + self.try_reserve(count)?; 1927 + let len = self.len(); 1928 + unsafe { ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count) }; 1929 + self.len += count; 1930 + Ok(()) 1931 + } 1932 + 1921 1933 /// Removes the specified range from the vector in bulk, returning all 1922 1934 /// removed elements as an iterator. If the iterator is dropped before 1923 1935 /// being fully consumed, it drops the remaining removed elements. ··· 2350 2340 } 2351 2341 } 2352 2342 2343 + /// Tries to resize the `Vec` in-place so that `len` is equal to `new_len`. 2344 + /// 2345 + /// If `new_len` is greater than `len`, the `Vec` is extended by the 2346 + /// difference, with each additional slot filled with `value`. 2347 + /// If `new_len` is less than `len`, the `Vec` is simply truncated. 2348 + /// 2349 + /// This method requires `T` to implement [`Clone`], 2350 + /// in order to be able to clone the passed value. 2351 + /// If you need more flexibility (or want to rely on [`Default`] instead of 2352 + /// [`Clone`]), use [`Vec::resize_with`]. 2353 + /// If you only need to resize to a smaller size, use [`Vec::truncate`]. 2354 + /// 2355 + /// # Examples 2356 + /// 2357 + /// ``` 2358 + /// let mut vec = vec!["hello"]; 2359 + /// vec.try_resize(3, "world").unwrap(); 2360 + /// assert_eq!(vec, ["hello", "world", "world"]); 2361 + /// 2362 + /// let mut vec = vec![1, 2, 3, 4]; 2363 + /// vec.try_resize(2, 0).unwrap(); 2364 + /// assert_eq!(vec, [1, 2]); 2365 + /// 2366 + /// let mut vec = vec![42]; 2367 + /// let result = vec.try_resize(usize::MAX, 0); 2368 + /// assert!(result.is_err()); 2369 + /// ``` 2370 + #[stable(feature = "kernel", since = "1.0.0")] 2371 + pub fn try_resize(&mut self, new_len: usize, value: T) -> Result<(), TryReserveError> { 2372 + let len = self.len(); 2373 + 2374 + if new_len > len { 2375 + self.try_extend_with(new_len - len, ExtendElement(value)) 2376 + } else { 2377 + self.truncate(new_len); 2378 + Ok(()) 2379 + } 2380 + } 2381 + 2353 2382 /// Clones and appends all elements in a slice to the `Vec`. 2354 2383 /// 2355 2384 /// Iterates over the slice `other`, clones each element, and then appends ··· 2412 2363 #[stable(feature = "vec_extend_from_slice", since = "1.6.0")] 2413 2364 pub fn extend_from_slice(&mut self, other: &[T]) { 2414 2365 self.spec_extend(other.iter()) 2366 + } 2367 + 2368 + /// Tries to clone and append all elements in a slice to the `Vec`. 2369 + /// 2370 + /// Iterates over the slice `other`, clones each element, and then appends 2371 + /// it to this `Vec`. The `other` slice is traversed in-order. 2372 + /// 2373 + /// Note that this function is same as [`extend`] except that it is 2374 + /// specialized to work with slices instead. If and when Rust gets 2375 + /// specialization this function will likely be deprecated (but still 2376 + /// available). 2377 + /// 2378 + /// # Examples 2379 + /// 2380 + /// ``` 2381 + /// let mut vec = vec![1]; 2382 + /// vec.try_extend_from_slice(&[2, 3, 4]).unwrap(); 2383 + /// assert_eq!(vec, [1, 2, 3, 4]); 2384 + /// ``` 2385 + /// 2386 + /// [`extend`]: Vec::extend 2387 + #[stable(feature = "kernel", since = "1.0.0")] 2388 + pub fn try_extend_from_slice(&mut self, other: &[T]) -> Result<(), TryReserveError> { 2389 + self.try_spec_extend(other.iter()) 2415 2390 } 2416 2391 2417 2392 /// Copies elements from `src` range to the end of the vector. ··· 2575 2502 } 2576 2503 2577 2504 // len set by scope guard 2505 + } 2506 + } 2507 + 2508 + /// Try to extend the vector by `n` values, using the given generator. 2509 + fn try_extend_with<E: ExtendWith<T>>(&mut self, n: usize, mut value: E) -> Result<(), TryReserveError> { 2510 + self.try_reserve(n)?; 2511 + 2512 + unsafe { 2513 + let mut ptr = self.as_mut_ptr().add(self.len()); 2514 + // Use SetLenOnDrop to work around bug where compiler 2515 + // might not realize the store through `ptr` through self.set_len() 2516 + // don't alias. 2517 + let mut local_len = SetLenOnDrop::new(&mut self.len); 2518 + 2519 + // Write all elements except the last one 2520 + for _ in 1..n { 2521 + ptr::write(ptr, value.next()); 2522 + ptr = ptr.offset(1); 2523 + // Increment the length in every step in case next() panics 2524 + local_len.increment_len(1); 2525 + } 2526 + 2527 + if n > 0 { 2528 + // We can write the last element directly without cloning needlessly 2529 + ptr::write(ptr, value.last()); 2530 + local_len.increment_len(1); 2531 + } 2532 + 2533 + // len set by scope guard 2534 + Ok(()) 2578 2535 } 2579 2536 } 2580 2537 } ··· 2939 2836 self.set_len(len + 1); 2940 2837 } 2941 2838 } 2839 + } 2840 + 2841 + // leaf method to which various SpecFrom/SpecExtend implementations delegate when 2842 + // they have no further optimizations to apply 2843 + fn try_extend_desugared<I: Iterator<Item = T>>(&mut self, mut iterator: I) -> Result<(), TryReserveError> { 2844 + // This is the case for a general iterator. 2845 + // 2846 + // This function should be the moral equivalent of: 2847 + // 2848 + // for item in iterator { 2849 + // self.push(item); 2850 + // } 2851 + while let Some(element) = iterator.next() { 2852 + let len = self.len(); 2853 + if len == self.capacity() { 2854 + let (lower, _) = iterator.size_hint(); 2855 + self.try_reserve(lower.saturating_add(1))?; 2856 + } 2857 + unsafe { 2858 + ptr::write(self.as_mut_ptr().add(len), element); 2859 + // Since next() executes user code which can panic we have to bump the length 2860 + // after each step. 2861 + // NB can't overflow since we would have had to alloc the address space 2862 + self.set_len(len + 1); 2863 + } 2864 + } 2865 + 2866 + Ok(()) 2942 2867 } 2943 2868 2944 2869 /// Creates a splicing iterator that replaces the specified range in the vector

+30

rust/alloc/vec/set_len_on_drop.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + // Set the length of the vec when the `SetLenOnDrop` value goes out of scope. 4 + // 5 + // The idea is: The length field in SetLenOnDrop is a local variable 6 + // that the optimizer will see does not alias with any stores through the Vec's data 7 + // pointer. This is a workaround for alias analysis issue #32155 8 + pub(super) struct SetLenOnDrop<'a> { 9 + len: &'a mut usize, 10 + local_len: usize, 11 + } 12 + 13 + impl<'a> SetLenOnDrop<'a> { 14 + #[inline] 15 + pub(super) fn new(len: &'a mut usize) -> Self { 16 + SetLenOnDrop { local_len: *len, len } 17 + } 18 + 19 + #[inline] 20 + pub(super) fn increment_len(&mut self, increment: usize) { 21 + self.local_len += increment; 22 + } 23 + } 24 + 25 + impl Drop for SetLenOnDrop<'_> { 26 + #[inline] 27 + fn drop(&mut self) { 28 + *self.len = self.local_len; 29 + } 30 + }

+174

rust/alloc/vec/spec_extend.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + use crate::alloc::Allocator; 4 + use crate::collections::{TryReserveError, TryReserveErrorKind}; 5 + use core::iter::TrustedLen; 6 + use core::ptr::{self}; 7 + use core::slice::{self}; 8 + 9 + use super::{IntoIter, SetLenOnDrop, Vec}; 10 + 11 + // Specialization trait used for Vec::extend 12 + #[cfg(not(no_global_oom_handling))] 13 + pub(super) trait SpecExtend<T, I> { 14 + fn spec_extend(&mut self, iter: I); 15 + } 16 + 17 + // Specialization trait used for Vec::try_extend 18 + pub(super) trait TrySpecExtend<T, I> { 19 + fn try_spec_extend(&mut self, iter: I) -> Result<(), TryReserveError>; 20 + } 21 + 22 + #[cfg(not(no_global_oom_handling))] 23 + impl<T, I, A: Allocator> SpecExtend<T, I> for Vec<T, A> 24 + where 25 + I: Iterator<Item = T>, 26 + { 27 + default fn spec_extend(&mut self, iter: I) { 28 + self.extend_desugared(iter) 29 + } 30 + } 31 + 32 + impl<T, I, A: Allocator> TrySpecExtend<T, I> for Vec<T, A> 33 + where 34 + I: Iterator<Item = T>, 35 + { 36 + default fn try_spec_extend(&mut self, iter: I) -> Result<(), TryReserveError> { 37 + self.try_extend_desugared(iter) 38 + } 39 + } 40 + 41 + #[cfg(not(no_global_oom_handling))] 42 + impl<T, I, A: Allocator> SpecExtend<T, I> for Vec<T, A> 43 + where 44 + I: TrustedLen<Item = T>, 45 + { 46 + default fn spec_extend(&mut self, iterator: I) { 47 + // This is the case for a TrustedLen iterator. 48 + let (low, high) = iterator.size_hint(); 49 + if let Some(additional) = high { 50 + debug_assert_eq!( 51 + low, 52 + additional, 53 + "TrustedLen iterator's size hint is not exact: {:?}", 54 + (low, high) 55 + ); 56 + self.reserve(additional); 57 + unsafe { 58 + let mut ptr = self.as_mut_ptr().add(self.len()); 59 + let mut local_len = SetLenOnDrop::new(&mut self.len); 60 + iterator.for_each(move |element| { 61 + ptr::write(ptr, element); 62 + ptr = ptr.offset(1); 63 + // Since the loop executes user code which can panic we have to bump the pointer 64 + // after each step. 65 + // NB can't overflow since we would have had to alloc the address space 66 + local_len.increment_len(1); 67 + }); 68 + } 69 + } else { 70 + // Per TrustedLen contract a `None` upper bound means that the iterator length 71 + // truly exceeds usize::MAX, which would eventually lead to a capacity overflow anyway. 72 + // Since the other branch already panics eagerly (via `reserve()`) we do the same here. 73 + // This avoids additional codegen for a fallback code path which would eventually 74 + // panic anyway. 75 + panic!("capacity overflow"); 76 + } 77 + } 78 + } 79 + 80 + impl<T, I, A: Allocator> TrySpecExtend<T, I> for Vec<T, A> 81 + where 82 + I: TrustedLen<Item = T>, 83 + { 84 + default fn try_spec_extend(&mut self, iterator: I) -> Result<(), TryReserveError> { 85 + // This is the case for a TrustedLen iterator. 86 + let (low, high) = iterator.size_hint(); 87 + if let Some(additional) = high { 88 + debug_assert_eq!( 89 + low, 90 + additional, 91 + "TrustedLen iterator's size hint is not exact: {:?}", 92 + (low, high) 93 + ); 94 + self.try_reserve(additional)?; 95 + unsafe { 96 + let mut ptr = self.as_mut_ptr().add(self.len()); 97 + let mut local_len = SetLenOnDrop::new(&mut self.len); 98 + iterator.for_each(move |element| { 99 + ptr::write(ptr, element); 100 + ptr = ptr.offset(1); 101 + // Since the loop executes user code which can panic we have to bump the pointer 102 + // after each step. 103 + // NB can't overflow since we would have had to alloc the address space 104 + local_len.increment_len(1); 105 + }); 106 + } 107 + Ok(()) 108 + } else { 109 + Err(TryReserveErrorKind::CapacityOverflow.into()) 110 + } 111 + } 112 + } 113 + 114 + #[cfg(not(no_global_oom_handling))] 115 + impl<T, A: Allocator> SpecExtend<T, IntoIter<T>> for Vec<T, A> { 116 + fn spec_extend(&mut self, mut iterator: IntoIter<T>) { 117 + unsafe { 118 + self.append_elements(iterator.as_slice() as _); 119 + } 120 + iterator.forget_remaining_elements(); 121 + } 122 + } 123 + 124 + impl<T, A: Allocator> TrySpecExtend<T, IntoIter<T>> for Vec<T, A> { 125 + fn try_spec_extend(&mut self, mut iterator: IntoIter<T>) -> Result<(), TryReserveError> { 126 + unsafe { 127 + self.try_append_elements(iterator.as_slice() as _)?; 128 + } 129 + iterator.forget_remaining_elements(); 130 + Ok(()) 131 + } 132 + } 133 + 134 + #[cfg(not(no_global_oom_handling))] 135 + impl<'a, T: 'a, I, A: Allocator + 'a> SpecExtend<&'a T, I> for Vec<T, A> 136 + where 137 + I: Iterator<Item = &'a T>, 138 + T: Clone, 139 + { 140 + default fn spec_extend(&mut self, iterator: I) { 141 + self.spec_extend(iterator.cloned()) 142 + } 143 + } 144 + 145 + impl<'a, T: 'a, I, A: Allocator + 'a> TrySpecExtend<&'a T, I> for Vec<T, A> 146 + where 147 + I: Iterator<Item = &'a T>, 148 + T: Clone, 149 + { 150 + default fn try_spec_extend(&mut self, iterator: I) -> Result<(), TryReserveError> { 151 + self.try_spec_extend(iterator.cloned()) 152 + } 153 + } 154 + 155 + #[cfg(not(no_global_oom_handling))] 156 + impl<'a, T: 'a, A: Allocator + 'a> SpecExtend<&'a T, slice::Iter<'a, T>> for Vec<T, A> 157 + where 158 + T: Copy, 159 + { 160 + fn spec_extend(&mut self, iterator: slice::Iter<'a, T>) { 161 + let slice = iterator.as_slice(); 162 + unsafe { self.append_elements(slice) }; 163 + } 164 + } 165 + 166 + impl<'a, T: 'a, A: Allocator + 'a> TrySpecExtend<&'a T, slice::Iter<'a, T>> for Vec<T, A> 167 + where 168 + T: Copy, 169 + { 170 + fn try_spec_extend(&mut self, iterator: slice::Iter<'a, T>) -> Result<(), TryReserveError> { 171 + let slice = iterator.as_slice(); 172 + unsafe { self.try_append_elements(slice) } 173 + } 174 + }

+2

rust/bindings/bindings_helper.h

··· 8 8 9 9 #include <linux/slab.h> 10 10 #include <linux/refcount.h> 11 + #include <linux/wait.h> 12 + #include <linux/sched.h> 11 13 12 14 /* `bindgen` gets confused at certain things. */ 13 15 const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL;

+82

rust/helpers.c

··· 20 20 21 21 #include <linux/bug.h> 22 22 #include <linux/build_bug.h> 23 + #include <linux/err.h> 23 24 #include <linux/refcount.h> 25 + #include <linux/mutex.h> 26 + #include <linux/spinlock.h> 27 + #include <linux/sched/signal.h> 28 + #include <linux/wait.h> 24 29 25 30 __noreturn void rust_helper_BUG(void) 26 31 { 27 32 BUG(); 28 33 } 29 34 EXPORT_SYMBOL_GPL(rust_helper_BUG); 35 + 36 + void rust_helper_mutex_lock(struct mutex *lock) 37 + { 38 + mutex_lock(lock); 39 + } 40 + EXPORT_SYMBOL_GPL(rust_helper_mutex_lock); 41 + 42 + void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, 43 + struct lock_class_key *key) 44 + { 45 + #ifdef CONFIG_DEBUG_SPINLOCK 46 + __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); 47 + #else 48 + spin_lock_init(lock); 49 + #endif 50 + } 51 + EXPORT_SYMBOL_GPL(rust_helper___spin_lock_init); 52 + 53 + void rust_helper_spin_lock(spinlock_t *lock) 54 + { 55 + spin_lock(lock); 56 + } 57 + EXPORT_SYMBOL_GPL(rust_helper_spin_lock); 58 + 59 + void rust_helper_spin_unlock(spinlock_t *lock) 60 + { 61 + spin_unlock(lock); 62 + } 63 + EXPORT_SYMBOL_GPL(rust_helper_spin_unlock); 64 + 65 + void rust_helper_init_wait(struct wait_queue_entry *wq_entry) 66 + { 67 + init_wait(wq_entry); 68 + } 69 + EXPORT_SYMBOL_GPL(rust_helper_init_wait); 70 + 71 + int rust_helper_signal_pending(struct task_struct *t) 72 + { 73 + return signal_pending(t); 74 + } 75 + EXPORT_SYMBOL_GPL(rust_helper_signal_pending); 30 76 31 77 refcount_t rust_helper_REFCOUNT_INIT(int n) 32 78 { ··· 91 45 return refcount_dec_and_test(r); 92 46 } 93 47 EXPORT_SYMBOL_GPL(rust_helper_refcount_dec_and_test); 48 + 49 + __force void *rust_helper_ERR_PTR(long err) 50 + { 51 + return ERR_PTR(err); 52 + } 53 + EXPORT_SYMBOL_GPL(rust_helper_ERR_PTR); 54 + 55 + bool rust_helper_IS_ERR(__force const void *ptr) 56 + { 57 + return IS_ERR(ptr); 58 + } 59 + EXPORT_SYMBOL_GPL(rust_helper_IS_ERR); 60 + 61 + long rust_helper_PTR_ERR(__force const void *ptr) 62 + { 63 + return PTR_ERR(ptr); 64 + } 65 + EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR); 66 + 67 + struct task_struct *rust_helper_get_current(void) 68 + { 69 + return current; 70 + } 71 + EXPORT_SYMBOL_GPL(rust_helper_get_current); 72 + 73 + void rust_helper_get_task_struct(struct task_struct *t) 74 + { 75 + get_task_struct(t); 76 + } 77 + EXPORT_SYMBOL_GPL(rust_helper_get_task_struct); 78 + 79 + void rust_helper_put_task_struct(struct task_struct *t) 80 + { 81 + put_task_struct(t); 82 + } 83 + EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); 94 84 95 85 /* 96 86 * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type

+136 -1

rust/kernel/error.rs

··· 72 72 pub struct Error(core::ffi::c_int); 73 73 74 74 impl Error { 75 + /// Creates an [`Error`] from a kernel error code. 76 + /// 77 + /// It is a bug to pass an out-of-range `errno`. `EINVAL` would 78 + /// be returned in such a case. 79 + pub(crate) fn from_errno(errno: core::ffi::c_int) -> Error { 80 + if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { 81 + // TODO: Make it a `WARN_ONCE` once available. 82 + crate::pr_warn!( 83 + "attempted to create `Error` with out of range `errno`: {}", 84 + errno 85 + ); 86 + return code::EINVAL; 87 + } 88 + 89 + // INVARIANT: The check above ensures the type invariant 90 + // will hold. 91 + Error(errno) 92 + } 93 + 94 + /// Creates an [`Error`] from a kernel error code. 95 + /// 96 + /// # Safety 97 + /// 98 + /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). 99 + unsafe fn from_errno_unchecked(errno: core::ffi::c_int) -> Error { 100 + // INVARIANT: The contract ensures the type invariant 101 + // will hold. 102 + Error(errno) 103 + } 104 + 75 105 /// Returns the kernel error code. 76 - pub fn to_kernel_errno(self) -> core::ffi::c_int { 106 + pub fn to_errno(self) -> core::ffi::c_int { 77 107 self.0 108 + } 109 + 110 + /// Returns the error encoded as a pointer. 111 + #[allow(dead_code)] 112 + pub(crate) fn to_ptr<T>(self) -> *mut T { 113 + // SAFETY: self.0 is a valid error due to its invariant. 114 + unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ } 78 115 } 79 116 } 80 117 ··· 178 141 /// it should still be modeled as returning a `Result` rather than 179 142 /// just an [`Error`]. 180 143 pub type Result<T = ()> = core::result::Result<T, Error>; 144 + 145 + /// Converts an integer as returned by a C kernel function to an error if it's negative, and 146 + /// `Ok(())` otherwise. 147 + pub fn to_result(err: core::ffi::c_int) -> Result { 148 + if err < 0 { 149 + Err(Error::from_errno(err)) 150 + } else { 151 + Ok(()) 152 + } 153 + } 154 + 155 + /// Transform a kernel "error pointer" to a normal pointer. 156 + /// 157 + /// Some kernel C API functions return an "error pointer" which optionally 158 + /// embeds an `errno`. Callers are supposed to check the returned pointer 159 + /// for errors. This function performs the check and converts the "error pointer" 160 + /// to a normal pointer in an idiomatic fashion. 161 + /// 162 + /// # Examples 163 + /// 164 + /// ```ignore 165 + /// # use kernel::from_err_ptr; 166 + /// # use kernel::bindings; 167 + /// fn devm_platform_ioremap_resource( 168 + /// pdev: &mut PlatformDevice, 169 + /// index: u32, 170 + /// ) -> Result<*mut core::ffi::c_void> { 171 + /// // SAFETY: FFI call. 172 + /// unsafe { 173 + /// from_err_ptr(bindings::devm_platform_ioremap_resource( 174 + /// pdev.to_ptr(), 175 + /// index, 176 + /// )) 177 + /// } 178 + /// } 179 + /// ``` 180 + // TODO: Remove `dead_code` marker once an in-kernel client is available. 181 + #[allow(dead_code)] 182 + pub(crate) fn from_err_ptr<T>(ptr: *mut T) -> Result<*mut T> { 183 + // CAST: Casting a pointer to `*const core::ffi::c_void` is always valid. 184 + let const_ptr: *const core::ffi::c_void = ptr.cast(); 185 + // SAFETY: The FFI function does not deref the pointer. 186 + if unsafe { bindings::IS_ERR(const_ptr) } { 187 + // SAFETY: The FFI function does not deref the pointer. 188 + let err = unsafe { bindings::PTR_ERR(const_ptr) }; 189 + // CAST: If `IS_ERR()` returns `true`, 190 + // then `PTR_ERR()` is guaranteed to return a 191 + // negative value greater-or-equal to `-bindings::MAX_ERRNO`, 192 + // which always fits in an `i16`, as per the invariant above. 193 + // And an `i16` always fits in an `i32`. So casting `err` to 194 + // an `i32` can never overflow, and is always valid. 195 + // 196 + // SAFETY: `IS_ERR()` ensures `err` is a 197 + // negative value greater-or-equal to `-bindings::MAX_ERRNO`. 198 + #[allow(clippy::unnecessary_cast)] 199 + return Err(unsafe { Error::from_errno_unchecked(err as core::ffi::c_int) }); 200 + } 201 + Ok(ptr) 202 + } 203 + 204 + /// Calls a closure returning a [`crate::error::Result<T>`] and converts the result to 205 + /// a C integer result. 206 + /// 207 + /// This is useful when calling Rust functions that return [`crate::error::Result<T>`] 208 + /// from inside `extern "C"` functions that need to return an integer error result. 209 + /// 210 + /// `T` should be convertible from an `i16` via `From<i16>`. 211 + /// 212 + /// # Examples 213 + /// 214 + /// ```ignore 215 + /// # use kernel::from_result; 216 + /// # use kernel::bindings; 217 + /// unsafe extern "C" fn probe_callback( 218 + /// pdev: *mut bindings::platform_device, 219 + /// ) -> core::ffi::c_int { 220 + /// from_result(|| { 221 + /// let ptr = devm_alloc(pdev)?; 222 + /// bindings::platform_set_drvdata(pdev, ptr); 223 + /// Ok(0) 224 + /// }) 225 + /// } 226 + /// ``` 227 + // TODO: Remove `dead_code` marker once an in-kernel client is available. 228 + #[allow(dead_code)] 229 + pub(crate) fn from_result<T, F>(f: F) -> T 230 + where 231 + T: From<i16>, 232 + F: FnOnce() -> Result<T>, 233 + { 234 + match f() { 235 + Ok(v) => v, 236 + // NO-OVERFLOW: negative `errno`s are no smaller than `-bindings::MAX_ERRNO`, 237 + // `-bindings::MAX_ERRNO` fits in an `i16` as per invariant above, 238 + // therefore a negative `errno` always fits in an `i16` and will not overflow. 239 + Err(e) => T::from(e.to_errno() as i16), 240 + } 241 + }

+1427

rust/kernel/init.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + //! API to safely and fallibly initialize pinned `struct`s using in-place constructors. 4 + //! 5 + //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack 6 + //! overflow. 7 + //! 8 + //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential 9 + //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. 10 + //! 11 + //! # Overview 12 + //! 13 + //! To initialize a `struct` with an in-place constructor you will need two things: 14 + //! - an in-place constructor, 15 + //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], 16 + //! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]). 17 + //! 18 + //! To get an in-place constructor there are generally three options: 19 + //! - directly creating an in-place constructor using the [`pin_init!`] macro, 20 + //! - a custom function/macro returning an in-place constructor provided by someone else, 21 + //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. 22 + //! 23 + //! Aside from pinned initialization, this API also supports in-place construction without pinning, 24 + //! the macros/types/functions are generally named like the pinned variants without the `pin` 25 + //! prefix. 26 + //! 27 + //! # Examples 28 + //! 29 + //! ## Using the [`pin_init!`] macro 30 + //! 31 + //! If you want to use [`PinInit`], then you will have to annotate your `struct` with 32 + //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for 33 + //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via 34 + //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is 35 + //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. 36 + //! 37 + //! ```rust 38 + //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 39 + //! use kernel::{prelude::*, sync::Mutex, new_mutex}; 40 + //! # use core::pin::Pin; 41 + //! #[pin_data] 42 + //! struct Foo { 43 + //! #[pin] 44 + //! a: Mutex<usize>, 45 + //! b: u32, 46 + //! } 47 + //! 48 + //! let foo = pin_init!(Foo { 49 + //! a <- new_mutex!(42, "Foo::a"), 50 + //! b: 24, 51 + //! }); 52 + //! ``` 53 + //! 54 + //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like 55 + //! (or just the stack) to actually initialize a `Foo`: 56 + //! 57 + //! ```rust 58 + //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 59 + //! # use kernel::{prelude::*, sync::Mutex, new_mutex}; 60 + //! # use core::pin::Pin; 61 + //! # #[pin_data] 62 + //! # struct Foo { 63 + //! # #[pin] 64 + //! # a: Mutex<usize>, 65 + //! # b: u32, 66 + //! # } 67 + //! # let foo = pin_init!(Foo { 68 + //! # a <- new_mutex!(42, "Foo::a"), 69 + //! # b: 24, 70 + //! # }); 71 + //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo); 72 + //! ``` 73 + //! 74 + //! For more information see the [`pin_init!`] macro. 75 + //! 76 + //! ## Using a custom function/macro that returns an initializer 77 + //! 78 + //! Many types from the kernel supply a function/macro that returns an initializer, because the 79 + //! above method only works for types where you can access the fields. 80 + //! 81 + //! ```rust 82 + //! # use kernel::{new_mutex, sync::{Arc, Mutex}}; 83 + //! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx")); 84 + //! ``` 85 + //! 86 + //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: 87 + //! 88 + //! ```rust 89 + //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 90 + //! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; 91 + //! #[pin_data] 92 + //! struct DriverData { 93 + //! #[pin] 94 + //! status: Mutex<i32>, 95 + //! buffer: Box<[u8; 1_000_000]>, 96 + //! } 97 + //! 98 + //! impl DriverData { 99 + //! fn new() -> impl PinInit<Self, Error> { 100 + //! try_pin_init!(Self { 101 + //! status <- new_mutex!(0, "DriverData::status"), 102 + //! buffer: Box::init(kernel::init::zeroed())?, 103 + //! }) 104 + //! } 105 + //! } 106 + //! ``` 107 + //! 108 + //! ## Manual creation of an initializer 109 + //! 110 + //! Often when working with primitives the previous approaches are not sufficient. That is where 111 + //! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a 112 + //! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure 113 + //! actually does the initialization in the correct way. Here are the things to look out for 114 + //! (we are calling the parameter to the closure `slot`): 115 + //! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so 116 + //! `slot` now contains a valid bit pattern for the type `T`, 117 + //! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so 118 + //! you need to take care to clean up anything if your initialization fails mid-way, 119 + //! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of 120 + //! `slot` gets called. 121 + //! 122 + //! ```rust 123 + //! use kernel::{prelude::*, init}; 124 + //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; 125 + //! # mod bindings { 126 + //! # pub struct foo; 127 + //! # pub unsafe fn init_foo(_ptr: *mut foo) {} 128 + //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} 129 + //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } 130 + //! # } 131 + //! /// # Invariants 132 + //! /// 133 + //! /// `foo` is always initialized 134 + //! #[pin_data(PinnedDrop)] 135 + //! pub struct RawFoo { 136 + //! #[pin] 137 + //! foo: Opaque<bindings::foo>, 138 + //! #[pin] 139 + //! _p: PhantomPinned, 140 + //! } 141 + //! 142 + //! impl RawFoo { 143 + //! pub fn new(flags: u32) -> impl PinInit<Self, Error> { 144 + //! // SAFETY: 145 + //! // - when the closure returns `Ok(())`, then it has successfully initialized and 146 + //! // enabled `foo`, 147 + //! // - when it returns `Err(e)`, then it has cleaned up before 148 + //! unsafe { 149 + //! init::pin_init_from_closure(move |slot: *mut Self| { 150 + //! // `slot` contains uninit memory, avoid creating a reference. 151 + //! let foo = addr_of_mut!((*slot).foo); 152 + //! 153 + //! // Initialize the `foo` 154 + //! bindings::init_foo(Opaque::raw_get(foo)); 155 + //! 156 + //! // Try to enable it. 157 + //! let err = bindings::enable_foo(Opaque::raw_get(foo), flags); 158 + //! if err != 0 { 159 + //! // Enabling has failed, first clean up the foo and then return the error. 160 + //! bindings::destroy_foo(Opaque::raw_get(foo)); 161 + //! return Err(Error::from_kernel_errno(err)); 162 + //! } 163 + //! 164 + //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. 165 + //! Ok(()) 166 + //! }) 167 + //! } 168 + //! } 169 + //! } 170 + //! 171 + //! #[pinned_drop] 172 + //! impl PinnedDrop for RawFoo { 173 + //! fn drop(self: Pin<&mut Self>) { 174 + //! // SAFETY: Since `foo` is initialized, destroying is safe. 175 + //! unsafe { bindings::destroy_foo(self.foo.get()) }; 176 + //! } 177 + //! } 178 + //! ``` 179 + //! 180 + //! For the special case where initializing a field is a single FFI-function call that cannot fail, 181 + //! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single 182 + //! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination 183 + //! with [`pin_init!`]. 184 + //! 185 + //! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside 186 + //! the `kernel` crate. The [`sync`] module is a good starting point. 187 + //! 188 + //! [`sync`]: kernel::sync 189 + //! [pinning]: https://doc.rust-lang.org/std/pin/index.html 190 + //! [structurally pinned fields]: 191 + //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field 192 + //! [stack]: crate::stack_pin_init 193 + //! [`Arc<T>`]: crate::sync::Arc 194 + //! [`impl PinInit<Foo>`]: PinInit 195 + //! [`impl PinInit<T, E>`]: PinInit 196 + //! [`impl Init<T, E>`]: Init 197 + //! [`Opaque`]: kernel::types::Opaque 198 + //! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init 199 + //! [`pin_data`]: ::macros::pin_data 200 + 201 + use crate::{ 202 + error::{self, Error}, 203 + sync::UniqueArc, 204 + }; 205 + use alloc::boxed::Box; 206 + use core::{ 207 + alloc::AllocError, 208 + cell::Cell, 209 + convert::Infallible, 210 + marker::PhantomData, 211 + mem::MaybeUninit, 212 + num::*, 213 + pin::Pin, 214 + ptr::{self, NonNull}, 215 + }; 216 + 217 + #[doc(hidden)] 218 + pub mod __internal; 219 + #[doc(hidden)] 220 + pub mod macros; 221 + 222 + /// Initialize and pin a type directly on the stack. 223 + /// 224 + /// # Examples 225 + /// 226 + /// ```rust 227 + /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 228 + /// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; 229 + /// # use macros::pin_data; 230 + /// # use core::pin::Pin; 231 + /// #[pin_data] 232 + /// struct Foo { 233 + /// #[pin] 234 + /// a: Mutex<usize>, 235 + /// b: Bar, 236 + /// } 237 + /// 238 + /// #[pin_data] 239 + /// struct Bar { 240 + /// x: u32, 241 + /// } 242 + /// 243 + /// stack_pin_init!(let foo = pin_init!(Foo { 244 + /// a <- new_mutex!(42), 245 + /// b: Bar { 246 + /// x: 64, 247 + /// }, 248 + /// })); 249 + /// let foo: Pin<&mut Foo> = foo; 250 + /// pr_info!("a: {}", &*foo.a.lock()); 251 + /// ``` 252 + /// 253 + /// # Syntax 254 + /// 255 + /// A normal `let` binding with optional type annotation. The expression is expected to implement 256 + /// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error 257 + /// type, then use [`stack_try_pin_init!`]. 258 + #[macro_export] 259 + macro_rules! stack_pin_init { 260 + (let $var:ident $(: $t:ty)? = $val:expr) => { 261 + let val = $val; 262 + let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); 263 + let mut $var = match $crate::init::__internal::StackInit::init($var, val) { 264 + Ok(res) => res, 265 + Err(x) => { 266 + let x: ::core::convert::Infallible = x; 267 + match x {} 268 + } 269 + }; 270 + }; 271 + } 272 + 273 + /// Initialize and pin a type directly on the stack. 274 + /// 275 + /// # Examples 276 + /// 277 + /// ```rust 278 + /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 279 + /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; 280 + /// # use macros::pin_data; 281 + /// # use core::{alloc::AllocError, pin::Pin}; 282 + /// #[pin_data] 283 + /// struct Foo { 284 + /// #[pin] 285 + /// a: Mutex<usize>, 286 + /// b: Box<Bar>, 287 + /// } 288 + /// 289 + /// struct Bar { 290 + /// x: u32, 291 + /// } 292 + /// 293 + /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo { 294 + /// a <- new_mutex!(42), 295 + /// b: Box::try_new(Bar { 296 + /// x: 64, 297 + /// })?, 298 + /// })); 299 + /// let foo = foo.unwrap(); 300 + /// pr_info!("a: {}", &*foo.a.lock()); 301 + /// ``` 302 + /// 303 + /// ```rust 304 + /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 305 + /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; 306 + /// # use macros::pin_data; 307 + /// # use core::{alloc::AllocError, pin::Pin}; 308 + /// #[pin_data] 309 + /// struct Foo { 310 + /// #[pin] 311 + /// a: Mutex<usize>, 312 + /// b: Box<Bar>, 313 + /// } 314 + /// 315 + /// struct Bar { 316 + /// x: u32, 317 + /// } 318 + /// 319 + /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { 320 + /// a <- new_mutex!(42), 321 + /// b: Box::try_new(Bar { 322 + /// x: 64, 323 + /// })?, 324 + /// })); 325 + /// pr_info!("a: {}", &*foo.a.lock()); 326 + /// # Ok::<_, AllocError>(()) 327 + /// ``` 328 + /// 329 + /// # Syntax 330 + /// 331 + /// A normal `let` binding with optional type annotation. The expression is expected to implement 332 + /// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the 333 + /// `=` will propagate this error. 334 + #[macro_export] 335 + macro_rules! stack_try_pin_init { 336 + (let $var:ident $(: $t:ty)? = $val:expr) => { 337 + let val = $val; 338 + let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); 339 + let mut $var = $crate::init::__internal::StackInit::init($var, val); 340 + }; 341 + (let $var:ident $(: $t:ty)? =? $val:expr) => { 342 + let val = $val; 343 + let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); 344 + let mut $var = $crate::init::__internal::StackInit::init($var, val)?; 345 + }; 346 + } 347 + 348 + /// Construct an in-place, pinned initializer for `struct`s. 349 + /// 350 + /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use 351 + /// [`try_pin_init!`]. 352 + /// 353 + /// The syntax is almost identical to that of a normal `struct` initializer: 354 + /// 355 + /// ```rust 356 + /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 357 + /// # use kernel::{init, pin_init, macros::pin_data, init::*}; 358 + /// # use core::pin::Pin; 359 + /// #[pin_data] 360 + /// struct Foo { 361 + /// a: usize, 362 + /// b: Bar, 363 + /// } 364 + /// 365 + /// #[pin_data] 366 + /// struct Bar { 367 + /// x: u32, 368 + /// } 369 + /// 370 + /// # fn demo() -> impl PinInit<Foo> { 371 + /// let a = 42; 372 + /// 373 + /// let initializer = pin_init!(Foo { 374 + /// a, 375 + /// b: Bar { 376 + /// x: 64, 377 + /// }, 378 + /// }); 379 + /// # initializer } 380 + /// # Box::pin_init(demo()).unwrap(); 381 + /// ``` 382 + /// 383 + /// Arbitrary Rust expressions can be used to set the value of a variable. 384 + /// 385 + /// The fields are initialized in the order that they appear in the initializer. So it is possible 386 + /// to read already initialized fields using raw pointers. 387 + /// 388 + /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the 389 + /// initializer. 390 + /// 391 + /// # Init-functions 392 + /// 393 + /// When working with this API it is often desired to let others construct your types without 394 + /// giving access to all fields. This is where you would normally write a plain function `new` 395 + /// that would return a new instance of your type. With this API that is also possible. 396 + /// However, there are a few extra things to keep in mind. 397 + /// 398 + /// To create an initializer function, simply declare it like this: 399 + /// 400 + /// ```rust 401 + /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 402 + /// # use kernel::{init, pin_init, prelude::*, init::*}; 403 + /// # use core::pin::Pin; 404 + /// # #[pin_data] 405 + /// # struct Foo { 406 + /// # a: usize, 407 + /// # b: Bar, 408 + /// # } 409 + /// # #[pin_data] 410 + /// # struct Bar { 411 + /// # x: u32, 412 + /// # } 413 + /// impl Foo { 414 + /// fn new() -> impl PinInit<Self> { 415 + /// pin_init!(Self { 416 + /// a: 42, 417 + /// b: Bar { 418 + /// x: 64, 419 + /// }, 420 + /// }) 421 + /// } 422 + /// } 423 + /// ``` 424 + /// 425 + /// Users of `Foo` can now create it like this: 426 + /// 427 + /// ```rust 428 + /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 429 + /// # use kernel::{init, pin_init, macros::pin_data, init::*}; 430 + /// # use core::pin::Pin; 431 + /// # #[pin_data] 432 + /// # struct Foo { 433 + /// # a: usize, 434 + /// # b: Bar, 435 + /// # } 436 + /// # #[pin_data] 437 + /// # struct Bar { 438 + /// # x: u32, 439 + /// # } 440 + /// # impl Foo { 441 + /// # fn new() -> impl PinInit<Self> { 442 + /// # pin_init!(Self { 443 + /// # a: 42, 444 + /// # b: Bar { 445 + /// # x: 64, 446 + /// # }, 447 + /// # }) 448 + /// # } 449 + /// # } 450 + /// let foo = Box::pin_init(Foo::new()); 451 + /// ``` 452 + /// 453 + /// They can also easily embed it into their own `struct`s: 454 + /// 455 + /// ```rust 456 + /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] 457 + /// # use kernel::{init, pin_init, macros::pin_data, init::*}; 458 + /// # use core::pin::Pin; 459 + /// # #[pin_data] 460 + /// # struct Foo { 461 + /// # a: usize, 462 + /// # b: Bar, 463 + /// # } 464 + /// # #[pin_data] 465 + /// # struct Bar { 466 + /// # x: u32, 467 + /// # } 468 + /// # impl Foo { 469 + /// # fn new() -> impl PinInit<Self> { 470 + /// # pin_init!(Self { 471 + /// # a: 42, 472 + /// # b: Bar { 473 + /// # x: 64, 474 + /// # }, 475 + /// # }) 476 + /// # } 477 + /// # } 478 + /// #[pin_data] 479 + /// struct FooContainer { 480 + /// #[pin] 481 + /// foo1: Foo, 482 + /// #[pin] 483 + /// foo2: Foo, 484 + /// other: u32, 485 + /// } 486 + /// 487 + /// impl FooContainer { 488 + /// fn new(other: u32) -> impl PinInit<Self> { 489 + /// pin_init!(Self { 490 + /// foo1 <- Foo::new(), 491 + /// foo2 <- Foo::new(), 492 + /// other, 493 + /// }) 494 + /// } 495 + /// } 496 + /// ``` 497 + /// 498 + /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. 499 + /// This signifies that the given field is initialized in-place. As with `struct` initializers, just 500 + /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. 501 + /// 502 + /// # Syntax 503 + /// 504 + /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with 505 + /// the following modifications is expected: 506 + /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. 507 + /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] 508 + /// pointer named `this` inside of the initializer. 509 + /// 510 + /// For instance: 511 + /// 512 + /// ```rust 513 + /// # use kernel::pin_init; 514 + /// # use macros::pin_data; 515 + /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; 516 + /// #[pin_data] 517 + /// struct Buf { 518 + /// // `ptr` points into `buf`. 519 + /// ptr: *mut u8, 520 + /// buf: [u8; 64], 521 + /// #[pin] 522 + /// pin: PhantomPinned, 523 + /// } 524 + /// pin_init!(&this in Buf { 525 + /// buf: [0; 64], 526 + /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, 527 + /// pin: PhantomPinned, 528 + /// }); 529 + /// ``` 530 + /// 531 + /// [`try_pin_init!`]: kernel::try_pin_init 532 + /// [`NonNull<Self>`]: core::ptr::NonNull 533 + // For a detailed example of how this macro works, see the module documentation of the hidden 534 + // module `__internal` inside of `init/__internal.rs`. 535 + #[macro_export] 536 + macro_rules! pin_init { 537 + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 538 + $($fields:tt)* 539 + }) => { 540 + $crate::try_pin_init!( 541 + @this($($this)?), 542 + @typ($t $(::<$($generics),*>)?), 543 + @fields($($fields)*), 544 + @error(::core::convert::Infallible), 545 + ) 546 + }; 547 + } 548 + 549 + /// Construct an in-place, fallible pinned initializer for `struct`s. 550 + /// 551 + /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. 552 + /// 553 + /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop 554 + /// initialization and return the error. 555 + /// 556 + /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when 557 + /// initialization fails, the memory can be safely deallocated without any further modifications. 558 + /// 559 + /// This macro defaults the error to [`Error`]. 560 + /// 561 + /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` 562 + /// after the `struct` initializer to specify the error type you want to use. 563 + /// 564 + /// # Examples 565 + /// 566 + /// ```rust 567 + /// # #![feature(new_uninit)] 568 + /// use kernel::{init::{self, PinInit}, error::Error}; 569 + /// #[pin_data] 570 + /// struct BigBuf { 571 + /// big: Box<[u8; 1024 * 1024 * 1024]>, 572 + /// small: [u8; 1024 * 1024], 573 + /// ptr: *mut u8, 574 + /// } 575 + /// 576 + /// impl BigBuf { 577 + /// fn new() -> impl PinInit<Self, Error> { 578 + /// try_pin_init!(Self { 579 + /// big: Box::init(init::zeroed())?, 580 + /// small: [0; 1024 * 1024], 581 + /// ptr: core::ptr::null_mut(), 582 + /// }? Error) 583 + /// } 584 + /// } 585 + /// ``` 586 + // For a detailed example of how this macro works, see the module documentation of the hidden 587 + // module `__internal` inside of `init/__internal.rs`. 588 + #[macro_export] 589 + macro_rules! try_pin_init { 590 + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 591 + $($fields:tt)* 592 + }) => { 593 + $crate::try_pin_init!( 594 + @this($($this)?), 595 + @typ($t $(::<$($generics),*>)? ), 596 + @fields($($fields)*), 597 + @error($crate::error::Error), 598 + ) 599 + }; 600 + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 601 + $($fields:tt)* 602 + }? $err:ty) => { 603 + $crate::try_pin_init!( 604 + @this($($this)?), 605 + @typ($t $(::<$($generics),*>)? ), 606 + @fields($($fields)*), 607 + @error($err), 608 + ) 609 + }; 610 + ( 611 + @this($($this:ident)?), 612 + @typ($t:ident $(::<$($generics:ty),*>)?), 613 + @fields($($fields:tt)*), 614 + @error($err:ty), 615 + ) => {{ 616 + // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return 617 + // type and shadow it later when we insert the arbitrary user code. That way there will be 618 + // no possibility of returning without `unsafe`. 619 + struct __InitOk; 620 + // Get the pin data from the supplied type. 621 + let data = unsafe { 622 + use $crate::init::__internal::HasPinData; 623 + $t$(::<$($generics),*>)?::__pin_data() 624 + }; 625 + // Ensure that `data` really is of type `PinData` and help with type inference: 626 + let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>( 627 + data, 628 + move |slot| { 629 + { 630 + // Shadow the structure so it cannot be used to return early. 631 + struct __InitOk; 632 + // Create the `this` so it can be referenced by the user inside of the 633 + // expressions creating the individual fields. 634 + $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? 635 + // Initialize every field. 636 + $crate::try_pin_init!(init_slot: 637 + @data(data), 638 + @slot(slot), 639 + @munch_fields($($fields)*,), 640 + ); 641 + // We use unreachable code to ensure that all fields have been mentioned exactly 642 + // once, this struct initializer will still be type-checked and complain with a 643 + // very natural error message if a field is forgotten/mentioned more than once. 644 + #[allow(unreachable_code, clippy::diverging_sub_expression)] 645 + if false { 646 + $crate::try_pin_init!(make_initializer: 647 + @slot(slot), 648 + @type_name($t), 649 + @munch_fields($($fields)*,), 650 + @acc(), 651 + ); 652 + } 653 + // Forget all guards, since initialization was a success. 654 + $crate::try_pin_init!(forget_guards: 655 + @munch_fields($($fields)*,), 656 + ); 657 + } 658 + Ok(__InitOk) 659 + } 660 + ); 661 + let init = move |slot| -> ::core::result::Result<(), $err> { 662 + init(slot).map(|__InitOk| ()) 663 + }; 664 + let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) }; 665 + init 666 + }}; 667 + (init_slot: 668 + @data($data:ident), 669 + @slot($slot:ident), 670 + @munch_fields($(,)?), 671 + ) => { 672 + // Endpoint of munching, no fields are left. 673 + }; 674 + (init_slot: 675 + @data($data:ident), 676 + @slot($slot:ident), 677 + // In-place initialization syntax. 678 + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 679 + ) => { 680 + let $field = $val; 681 + // Call the initializer. 682 + // 683 + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 684 + // return when an error/panic occurs. 685 + // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. 686 + unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? }; 687 + // Create the drop guard. 688 + // 689 + // We only give access to `&DropGuard`, so it cannot be forgotten via safe code. 690 + // 691 + // SAFETY: We forget the guard later when initialization has succeeded. 692 + let $field = &unsafe { 693 + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 694 + }; 695 + 696 + $crate::try_pin_init!(init_slot: 697 + @data($data), 698 + @slot($slot), 699 + @munch_fields($($rest)*), 700 + ); 701 + }; 702 + (init_slot: 703 + @data($data:ident), 704 + @slot($slot:ident), 705 + // Direct value init, this is safe for every field. 706 + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 707 + ) => { 708 + $(let $field = $val;)? 709 + // Initialize the field. 710 + // 711 + // SAFETY: The memory at `slot` is uninitialized. 712 + unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; 713 + // Create the drop guard: 714 + // 715 + // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. 716 + // 717 + // SAFETY: We forget the guard later when initialization has succeeded. 718 + let $field = &unsafe { 719 + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 720 + }; 721 + 722 + $crate::try_pin_init!(init_slot: 723 + @data($data), 724 + @slot($slot), 725 + @munch_fields($($rest)*), 726 + ); 727 + }; 728 + (make_initializer: 729 + @slot($slot:ident), 730 + @type_name($t:ident), 731 + @munch_fields($(,)?), 732 + @acc($($acc:tt)*), 733 + ) => { 734 + // Endpoint, nothing more to munch, create the initializer. 735 + // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to 736 + // get the correct type inference here: 737 + unsafe { 738 + ::core::ptr::write($slot, $t { 739 + $($acc)* 740 + }); 741 + } 742 + }; 743 + (make_initializer: 744 + @slot($slot:ident), 745 + @type_name($t:ident), 746 + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 747 + @acc($($acc:tt)*), 748 + ) => { 749 + $crate::try_pin_init!(make_initializer: 750 + @slot($slot), 751 + @type_name($t), 752 + @munch_fields($($rest)*), 753 + @acc($($acc)* $field: ::core::panic!(),), 754 + ); 755 + }; 756 + (make_initializer: 757 + @slot($slot:ident), 758 + @type_name($t:ident), 759 + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 760 + @acc($($acc:tt)*), 761 + ) => { 762 + $crate::try_pin_init!(make_initializer: 763 + @slot($slot), 764 + @type_name($t), 765 + @munch_fields($($rest)*), 766 + @acc($($acc)* $field: ::core::panic!(),), 767 + ); 768 + }; 769 + (forget_guards: 770 + @munch_fields($(,)?), 771 + ) => { 772 + // Munching finished. 773 + }; 774 + (forget_guards: 775 + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 776 + ) => { 777 + unsafe { $crate::init::__internal::DropGuard::forget($field) }; 778 + 779 + $crate::try_pin_init!(forget_guards: 780 + @munch_fields($($rest)*), 781 + ); 782 + }; 783 + (forget_guards: 784 + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 785 + ) => { 786 + unsafe { $crate::init::__internal::DropGuard::forget($field) }; 787 + 788 + $crate::try_pin_init!(forget_guards: 789 + @munch_fields($($rest)*), 790 + ); 791 + }; 792 + } 793 + 794 + /// Construct an in-place initializer for `struct`s. 795 + /// 796 + /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use 797 + /// [`try_init!`]. 798 + /// 799 + /// The syntax is identical to [`pin_init!`] and its safety caveats also apply: 800 + /// - `unsafe` code must guarantee either full initialization or return an error and allow 801 + /// deallocation of the memory. 802 + /// - the fields are initialized in the order given in the initializer. 803 + /// - no references to fields are allowed to be created inside of the initializer. 804 + /// 805 + /// This initializer is for initializing data in-place that might later be moved. If you want to 806 + /// pin-initialize, use [`pin_init!`]. 807 + // For a detailed example of how this macro works, see the module documentation of the hidden 808 + // module `__internal` inside of `init/__internal.rs`. 809 + #[macro_export] 810 + macro_rules! init { 811 + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 812 + $($fields:tt)* 813 + }) => { 814 + $crate::try_init!( 815 + @this($($this)?), 816 + @typ($t $(::<$($generics),*>)?), 817 + @fields($($fields)*), 818 + @error(::core::convert::Infallible), 819 + ) 820 + } 821 + } 822 + 823 + /// Construct an in-place fallible initializer for `struct`s. 824 + /// 825 + /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use 826 + /// [`init!`]. 827 + /// 828 + /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, 829 + /// append `? $type` after the `struct` initializer. 830 + /// The safety caveats from [`try_pin_init!`] also apply: 831 + /// - `unsafe` code must guarantee either full initialization or return an error and allow 832 + /// deallocation of the memory. 833 + /// - the fields are initialized in the order given in the initializer. 834 + /// - no references to fields are allowed to be created inside of the initializer. 835 + /// 836 + /// # Examples 837 + /// 838 + /// ```rust 839 + /// use kernel::{init::PinInit, error::Error, InPlaceInit}; 840 + /// struct BigBuf { 841 + /// big: Box<[u8; 1024 * 1024 * 1024]>, 842 + /// small: [u8; 1024 * 1024], 843 + /// } 844 + /// 845 + /// impl BigBuf { 846 + /// fn new() -> impl Init<Self, Error> { 847 + /// try_init!(Self { 848 + /// big: Box::init(zeroed())?, 849 + /// small: [0; 1024 * 1024], 850 + /// }? Error) 851 + /// } 852 + /// } 853 + /// ``` 854 + // For a detailed example of how this macro works, see the module documentation of the hidden 855 + // module `__internal` inside of `init/__internal.rs`. 856 + #[macro_export] 857 + macro_rules! try_init { 858 + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 859 + $($fields:tt)* 860 + }) => { 861 + $crate::try_init!( 862 + @this($($this)?), 863 + @typ($t $(::<$($generics),*>)?), 864 + @fields($($fields)*), 865 + @error($crate::error::Error), 866 + ) 867 + }; 868 + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { 869 + $($fields:tt)* 870 + }? $err:ty) => { 871 + $crate::try_init!( 872 + @this($($this)?), 873 + @typ($t $(::<$($generics),*>)?), 874 + @fields($($fields)*), 875 + @error($err), 876 + ) 877 + }; 878 + ( 879 + @this($($this:ident)?), 880 + @typ($t:ident $(::<$($generics:ty),*>)?), 881 + @fields($($fields:tt)*), 882 + @error($err:ty), 883 + ) => {{ 884 + // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return 885 + // type and shadow it later when we insert the arbitrary user code. That way there will be 886 + // no possibility of returning without `unsafe`. 887 + struct __InitOk; 888 + // Get the init data from the supplied type. 889 + let data = unsafe { 890 + use $crate::init::__internal::HasInitData; 891 + $t$(::<$($generics),*>)?::__init_data() 892 + }; 893 + // Ensure that `data` really is of type `InitData` and help with type inference: 894 + let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>( 895 + data, 896 + move |slot| { 897 + { 898 + // Shadow the structure so it cannot be used to return early. 899 + struct __InitOk; 900 + // Create the `this` so it can be referenced by the user inside of the 901 + // expressions creating the individual fields. 902 + $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? 903 + // Initialize every field. 904 + $crate::try_init!(init_slot: 905 + @slot(slot), 906 + @munch_fields($($fields)*,), 907 + ); 908 + // We use unreachable code to ensure that all fields have been mentioned exactly 909 + // once, this struct initializer will still be type-checked and complain with a 910 + // very natural error message if a field is forgotten/mentioned more than once. 911 + #[allow(unreachable_code, clippy::diverging_sub_expression)] 912 + if false { 913 + $crate::try_init!(make_initializer: 914 + @slot(slot), 915 + @type_name($t), 916 + @munch_fields($($fields)*,), 917 + @acc(), 918 + ); 919 + } 920 + // Forget all guards, since initialization was a success. 921 + $crate::try_init!(forget_guards: 922 + @munch_fields($($fields)*,), 923 + ); 924 + } 925 + Ok(__InitOk) 926 + } 927 + ); 928 + let init = move |slot| -> ::core::result::Result<(), $err> { 929 + init(slot).map(|__InitOk| ()) 930 + }; 931 + let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) }; 932 + init 933 + }}; 934 + (init_slot: 935 + @slot($slot:ident), 936 + @munch_fields( $(,)?), 937 + ) => { 938 + // Endpoint of munching, no fields are left. 939 + }; 940 + (init_slot: 941 + @slot($slot:ident), 942 + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 943 + ) => { 944 + let $field = $val; 945 + // Call the initializer. 946 + // 947 + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 948 + // return when an error/panic occurs. 949 + unsafe { 950 + $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; 951 + } 952 + // Create the drop guard. 953 + // 954 + // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. 955 + // 956 + // SAFETY: We forget the guard later when initialization has succeeded. 957 + let $field = &unsafe { 958 + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 959 + }; 960 + 961 + $crate::try_init!(init_slot: 962 + @slot($slot), 963 + @munch_fields($($rest)*), 964 + ); 965 + }; 966 + (init_slot: 967 + @slot($slot:ident), 968 + // Direct value init. 969 + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 970 + ) => { 971 + $(let $field = $val;)? 972 + // Call the initializer. 973 + // 974 + // SAFETY: The memory at `slot` is uninitialized. 975 + unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; 976 + // Create the drop guard. 977 + // 978 + // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. 979 + // 980 + // SAFETY: We forget the guard later when initialization has succeeded. 981 + let $field = &unsafe { 982 + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 983 + }; 984 + 985 + $crate::try_init!(init_slot: 986 + @slot($slot), 987 + @munch_fields($($rest)*), 988 + ); 989 + }; 990 + (make_initializer: 991 + @slot($slot:ident), 992 + @type_name($t:ident), 993 + @munch_fields( $(,)?), 994 + @acc($($acc:tt)*), 995 + ) => { 996 + // Endpoint, nothing more to munch, create the initializer. 997 + // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to 998 + // get the correct type inference here: 999 + unsafe { 1000 + ::core::ptr::write($slot, $t { 1001 + $($acc)* 1002 + }); 1003 + } 1004 + }; 1005 + (make_initializer: 1006 + @slot($slot:ident), 1007 + @type_name($t:ident), 1008 + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1009 + @acc($($acc:tt)*), 1010 + ) => { 1011 + $crate::try_init!(make_initializer: 1012 + @slot($slot), 1013 + @type_name($t), 1014 + @munch_fields($($rest)*), 1015 + @acc($($acc)*$field: ::core::panic!(),), 1016 + ); 1017 + }; 1018 + (make_initializer: 1019 + @slot($slot:ident), 1020 + @type_name($t:ident), 1021 + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1022 + @acc($($acc:tt)*), 1023 + ) => { 1024 + $crate::try_init!(make_initializer: 1025 + @slot($slot), 1026 + @type_name($t), 1027 + @munch_fields($($rest)*), 1028 + @acc($($acc)*$field: ::core::panic!(),), 1029 + ); 1030 + }; 1031 + (forget_guards: 1032 + @munch_fields($(,)?), 1033 + ) => { 1034 + // Munching finished. 1035 + }; 1036 + (forget_guards: 1037 + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1038 + ) => { 1039 + unsafe { $crate::init::__internal::DropGuard::forget($field) }; 1040 + 1041 + $crate::try_init!(forget_guards: 1042 + @munch_fields($($rest)*), 1043 + ); 1044 + }; 1045 + (forget_guards: 1046 + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1047 + ) => { 1048 + unsafe { $crate::init::__internal::DropGuard::forget($field) }; 1049 + 1050 + $crate::try_init!(forget_guards: 1051 + @munch_fields($($rest)*), 1052 + ); 1053 + }; 1054 + } 1055 + 1056 + /// A pin-initializer for the type `T`. 1057 + /// 1058 + /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can 1059 + /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the 1060 + /// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this. 1061 + /// 1062 + /// Also see the [module description](self). 1063 + /// 1064 + /// # Safety 1065 + /// 1066 + /// When implementing this type you will need to take great care. Also there are probably very few 1067 + /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. 1068 + /// 1069 + /// The [`PinInit::__pinned_init`] function 1070 + /// - returns `Ok(())` if it initialized every field of `slot`, 1071 + /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: 1072 + /// - `slot` can be deallocated without UB occurring, 1073 + /// - `slot` does not need to be dropped, 1074 + /// - `slot` is not partially initialized. 1075 + /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. 1076 + /// 1077 + /// [`Arc<T>`]: crate::sync::Arc 1078 + /// [`Arc::pin_init`]: crate::sync::Arc::pin_init 1079 + #[must_use = "An initializer must be used in order to create its value."] 1080 + pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { 1081 + /// Initializes `slot`. 1082 + /// 1083 + /// # Safety 1084 + /// 1085 + /// - `slot` is a valid pointer to uninitialized memory. 1086 + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to 1087 + /// deallocate. 1088 + /// - `slot` will not move until it is dropped, i.e. it will be pinned. 1089 + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; 1090 + } 1091 + 1092 + /// An initializer for `T`. 1093 + /// 1094 + /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can 1095 + /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the 1096 + /// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because 1097 + /// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well. 1098 + /// 1099 + /// Also see the [module description](self). 1100 + /// 1101 + /// # Safety 1102 + /// 1103 + /// When implementing this type you will need to take great care. Also there are probably very few 1104 + /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. 1105 + /// 1106 + /// The [`Init::__init`] function 1107 + /// - returns `Ok(())` if it initialized every field of `slot`, 1108 + /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: 1109 + /// - `slot` can be deallocated without UB occurring, 1110 + /// - `slot` does not need to be dropped, 1111 + /// - `slot` is not partially initialized. 1112 + /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. 1113 + /// 1114 + /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same 1115 + /// code as `__init`. 1116 + /// 1117 + /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to 1118 + /// move the pointee after initialization. 1119 + /// 1120 + /// [`Arc<T>`]: crate::sync::Arc 1121 + #[must_use = "An initializer must be used in order to create its value."] 1122 + pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized { 1123 + /// Initializes `slot`. 1124 + /// 1125 + /// # Safety 1126 + /// 1127 + /// - `slot` is a valid pointer to uninitialized memory. 1128 + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to 1129 + /// deallocate. 1130 + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; 1131 + } 1132 + 1133 + // SAFETY: Every in-place initializer can also be used as a pin-initializer. 1134 + unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I 1135 + where 1136 + I: Init<T, E>, 1137 + { 1138 + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { 1139 + // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not 1140 + // require `slot` to not move after init. 1141 + unsafe { self.__init(slot) } 1142 + } 1143 + } 1144 + 1145 + /// Creates a new [`PinInit<T, E>`] from the given closure. 1146 + /// 1147 + /// # Safety 1148 + /// 1149 + /// The closure: 1150 + /// - returns `Ok(())` if it initialized every field of `slot`, 1151 + /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: 1152 + /// - `slot` can be deallocated without UB occurring, 1153 + /// - `slot` does not need to be dropped, 1154 + /// - `slot` is not partially initialized. 1155 + /// - may assume that the `slot` does not move if `T: !Unpin`, 1156 + /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. 1157 + #[inline] 1158 + pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( 1159 + f: impl FnOnce(*mut T) -> Result<(), E>, 1160 + ) -> impl PinInit<T, E> { 1161 + __internal::InitClosure(f, PhantomData) 1162 + } 1163 + 1164 + /// Creates a new [`Init<T, E>`] from the given closure. 1165 + /// 1166 + /// # Safety 1167 + /// 1168 + /// The closure: 1169 + /// - returns `Ok(())` if it initialized every field of `slot`, 1170 + /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: 1171 + /// - `slot` can be deallocated without UB occurring, 1172 + /// - `slot` does not need to be dropped, 1173 + /// - `slot` is not partially initialized. 1174 + /// - the `slot` may move after initialization. 1175 + /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. 1176 + #[inline] 1177 + pub const unsafe fn init_from_closure<T: ?Sized, E>( 1178 + f: impl FnOnce(*mut T) -> Result<(), E>, 1179 + ) -> impl Init<T, E> { 1180 + __internal::InitClosure(f, PhantomData) 1181 + } 1182 + 1183 + /// An initializer that leaves the memory uninitialized. 1184 + /// 1185 + /// The initializer is a no-op. The `slot` memory is not changed. 1186 + #[inline] 1187 + pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { 1188 + // SAFETY: The memory is allowed to be uninitialized. 1189 + unsafe { init_from_closure(|_| Ok(())) } 1190 + } 1191 + 1192 + // SAFETY: Every type can be initialized by-value. 1193 + unsafe impl<T, E> Init<T, E> for T { 1194 + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { 1195 + unsafe { slot.write(self) }; 1196 + Ok(()) 1197 + } 1198 + } 1199 + 1200 + /// Smart pointer that can initialize memory in-place. 1201 + pub trait InPlaceInit<T>: Sized { 1202 + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this 1203 + /// type. 1204 + /// 1205 + /// If `T: !Unpin` it will not be able to move afterwards. 1206 + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> 1207 + where 1208 + E: From<AllocError>; 1209 + 1210 + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this 1211 + /// type. 1212 + /// 1213 + /// If `T: !Unpin` it will not be able to move afterwards. 1214 + fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>> 1215 + where 1216 + Error: From<E>, 1217 + { 1218 + // SAFETY: We delegate to `init` and only change the error type. 1219 + let init = unsafe { 1220 + pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) 1221 + }; 1222 + Self::try_pin_init(init) 1223 + } 1224 + 1225 + /// Use the given initializer to in-place initialize a `T`. 1226 + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> 1227 + where 1228 + E: From<AllocError>; 1229 + 1230 + /// Use the given initializer to in-place initialize a `T`. 1231 + fn init<E>(init: impl Init<T, E>) -> error::Result<Self> 1232 + where 1233 + Error: From<E>, 1234 + { 1235 + // SAFETY: We delegate to `init` and only change the error type. 1236 + let init = unsafe { 1237 + init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) 1238 + }; 1239 + Self::try_init(init) 1240 + } 1241 + } 1242 + 1243 + impl<T> InPlaceInit<T> for Box<T> { 1244 + #[inline] 1245 + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> 1246 + where 1247 + E: From<AllocError>, 1248 + { 1249 + let mut this = Box::try_new_uninit()?; 1250 + let slot = this.as_mut_ptr(); 1251 + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, 1252 + // slot is valid and will not be moved, because we pin it later. 1253 + unsafe { init.__pinned_init(slot)? }; 1254 + // SAFETY: All fields have been initialized. 1255 + Ok(unsafe { this.assume_init() }.into()) 1256 + } 1257 + 1258 + #[inline] 1259 + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> 1260 + where 1261 + E: From<AllocError>, 1262 + { 1263 + let mut this = Box::try_new_uninit()?; 1264 + let slot = this.as_mut_ptr(); 1265 + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, 1266 + // slot is valid. 1267 + unsafe { init.__init(slot)? }; 1268 + // SAFETY: All fields have been initialized. 1269 + Ok(unsafe { this.assume_init() }) 1270 + } 1271 + } 1272 + 1273 + impl<T> InPlaceInit<T> for UniqueArc<T> { 1274 + #[inline] 1275 + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> 1276 + where 1277 + E: From<AllocError>, 1278 + { 1279 + let mut this = UniqueArc::try_new_uninit()?; 1280 + let slot = this.as_mut_ptr(); 1281 + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, 1282 + // slot is valid and will not be moved, because we pin it later. 1283 + unsafe { init.__pinned_init(slot)? }; 1284 + // SAFETY: All fields have been initialized. 1285 + Ok(unsafe { this.assume_init() }.into()) 1286 + } 1287 + 1288 + #[inline] 1289 + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> 1290 + where 1291 + E: From<AllocError>, 1292 + { 1293 + let mut this = UniqueArc::try_new_uninit()?; 1294 + let slot = this.as_mut_ptr(); 1295 + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, 1296 + // slot is valid. 1297 + unsafe { init.__init(slot)? }; 1298 + // SAFETY: All fields have been initialized. 1299 + Ok(unsafe { this.assume_init() }) 1300 + } 1301 + } 1302 + 1303 + /// Trait facilitating pinned destruction. 1304 + /// 1305 + /// Use [`pinned_drop`] to implement this trait safely: 1306 + /// 1307 + /// ```rust 1308 + /// # use kernel::sync::Mutex; 1309 + /// use kernel::macros::pinned_drop; 1310 + /// use core::pin::Pin; 1311 + /// #[pin_data(PinnedDrop)] 1312 + /// struct Foo { 1313 + /// #[pin] 1314 + /// mtx: Mutex<usize>, 1315 + /// } 1316 + /// 1317 + /// #[pinned_drop] 1318 + /// impl PinnedDrop for Foo { 1319 + /// fn drop(self: Pin<&mut Self>) { 1320 + /// pr_info!("Foo is being dropped!"); 1321 + /// } 1322 + /// } 1323 + /// ``` 1324 + /// 1325 + /// # Safety 1326 + /// 1327 + /// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. 1328 + /// 1329 + /// [`pinned_drop`]: kernel::macros::pinned_drop 1330 + pub unsafe trait PinnedDrop: __internal::HasPinData { 1331 + /// Executes the pinned destructor of this type. 1332 + /// 1333 + /// While this function is marked safe, it is actually unsafe to call it manually. For this 1334 + /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code 1335 + /// and thus prevents this function from being called where it should not. 1336 + /// 1337 + /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute 1338 + /// automatically. 1339 + fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); 1340 + } 1341 + 1342 + /// Marker trait for types that can be initialized by writing just zeroes. 1343 + /// 1344 + /// # Safety 1345 + /// 1346 + /// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, 1347 + /// this is not UB: 1348 + /// 1349 + /// ```rust,ignore 1350 + /// let val: Self = unsafe { core::mem::zeroed() }; 1351 + /// ``` 1352 + pub unsafe trait Zeroable {} 1353 + 1354 + /// Create a new zeroed T. 1355 + /// 1356 + /// The returned initializer will write `0x00` to every byte of the given `slot`. 1357 + #[inline] 1358 + pub fn zeroed<T: Zeroable>() -> impl Init<T> { 1359 + // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` 1360 + // and because we write all zeroes, the memory is initialized. 1361 + unsafe { 1362 + init_from_closure(|slot: *mut T| { 1363 + slot.write_bytes(0, 1); 1364 + Ok(()) 1365 + }) 1366 + } 1367 + } 1368 + 1369 + macro_rules! impl_zeroable { 1370 + ($($({$($generics:tt)*})? $t:ty, )*) => { 1371 + $(unsafe impl$($($generics)*)? Zeroable for $t {})* 1372 + }; 1373 + } 1374 + 1375 + impl_zeroable! { 1376 + // SAFETY: All primitives that are allowed to be zero. 1377 + bool, 1378 + char, 1379 + u8, u16, u32, u64, u128, usize, 1380 + i8, i16, i32, i64, i128, isize, 1381 + f32, f64, 1382 + 1383 + // SAFETY: These are ZSTs, there is nothing to zero. 1384 + {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (), 1385 + 1386 + // SAFETY: Type is allowed to take any value, including all zeros. 1387 + {<T>} MaybeUninit<T>, 1388 + 1389 + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). 1390 + Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, 1391 + Option<NonZeroU128>, Option<NonZeroUsize>, 1392 + Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, 1393 + Option<NonZeroI128>, Option<NonZeroIsize>, 1394 + 1395 + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). 1396 + // 1397 + // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. 1398 + {<T: ?Sized>} Option<NonNull<T>>, 1399 + {<T: ?Sized>} Option<Box<T>>, 1400 + 1401 + // SAFETY: `null` pointer is valid. 1402 + // 1403 + // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be 1404 + // null. 1405 + // 1406 + // When `Pointee` gets stabilized, we could use 1407 + // `T: ?Sized where <T as Pointee>::Metadata: Zeroable` 1408 + {<T>} *mut T, {<T>} *const T, 1409 + 1410 + // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be 1411 + // zero. 1412 + {<T>} *mut [T], {<T>} *const [T], *mut str, *const str, 1413 + 1414 + // SAFETY: `T` is `Zeroable`. 1415 + {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>, 1416 + } 1417 + 1418 + macro_rules! impl_tuple_zeroable { 1419 + ($(,)?) => {}; 1420 + ($first:ident, $($t:ident),* $(,)?) => { 1421 + // SAFETY: All elements are zeroable and padding can be zero. 1422 + unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} 1423 + impl_tuple_zeroable!($($t),* ,); 1424 + } 1425 + } 1426 + 1427 + impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);

+235

rust/kernel/init/__internal.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + //! This module contains API-internal items for pin-init. 4 + //! 5 + //! These items must not be used outside of 6 + //! - `kernel/init.rs` 7 + //! - `macros/pin_data.rs` 8 + //! - `macros/pinned_drop.rs` 9 + 10 + use super::*; 11 + 12 + /// See the [nomicon] for what subtyping is. See also [this table]. 13 + /// 14 + /// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html 15 + /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns 16 + type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; 17 + 18 + /// This is the module-internal type implementing `PinInit` and `Init`. It is unsafe to create this 19 + /// type, since the closure needs to fulfill the same safety requirement as the 20 + /// `__pinned_init`/`__init` functions. 21 + pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>); 22 + 23 + // SAFETY: While constructing the `InitClosure`, the user promised that it upholds the 24 + // `__init` invariants. 25 + unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E> 26 + where 27 + F: FnOnce(*mut T) -> Result<(), E>, 28 + { 29 + #[inline] 30 + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { 31 + (self.0)(slot) 32 + } 33 + } 34 + 35 + /// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate 36 + /// the pin projections within the initializers. 37 + /// 38 + /// # Safety 39 + /// 40 + /// Only the `init` module is allowed to use this trait. 41 + pub unsafe trait HasPinData { 42 + type PinData: PinData; 43 + 44 + unsafe fn __pin_data() -> Self::PinData; 45 + } 46 + 47 + /// Marker trait for pinning data of structs. 48 + /// 49 + /// # Safety 50 + /// 51 + /// Only the `init` module is allowed to use this trait. 52 + pub unsafe trait PinData: Copy { 53 + type Datee: ?Sized + HasPinData; 54 + 55 + /// Type inference helper function. 56 + fn make_closure<F, O, E>(self, f: F) -> F 57 + where 58 + F: FnOnce(*mut Self::Datee) -> Result<O, E>, 59 + { 60 + f 61 + } 62 + } 63 + 64 + /// This trait is automatically implemented for every type. It aims to provide the same type 65 + /// inference help as `HasPinData`. 66 + /// 67 + /// # Safety 68 + /// 69 + /// Only the `init` module is allowed to use this trait. 70 + pub unsafe trait HasInitData { 71 + type InitData: InitData; 72 + 73 + unsafe fn __init_data() -> Self::InitData; 74 + } 75 + 76 + /// Same function as `PinData`, but for arbitrary data. 77 + /// 78 + /// # Safety 79 + /// 80 + /// Only the `init` module is allowed to use this trait. 81 + pub unsafe trait InitData: Copy { 82 + type Datee: ?Sized + HasInitData; 83 + 84 + /// Type inference helper function. 85 + fn make_closure<F, O, E>(self, f: F) -> F 86 + where 87 + F: FnOnce(*mut Self::Datee) -> Result<O, E>, 88 + { 89 + f 90 + } 91 + } 92 + 93 + pub struct AllData<T: ?Sized>(PhantomData<fn(Box<T>) -> Box<T>>); 94 + 95 + impl<T: ?Sized> Clone for AllData<T> { 96 + fn clone(&self) -> Self { 97 + *self 98 + } 99 + } 100 + 101 + impl<T: ?Sized> Copy for AllData<T> {} 102 + 103 + unsafe impl<T: ?Sized> InitData for AllData<T> { 104 + type Datee = T; 105 + } 106 + 107 + unsafe impl<T: ?Sized> HasInitData for T { 108 + type InitData = AllData<T>; 109 + 110 + unsafe fn __init_data() -> Self::InitData { 111 + AllData(PhantomData) 112 + } 113 + } 114 + 115 + /// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive. 116 + /// 117 + /// # Invariants 118 + /// 119 + /// If `self.is_init` is true, then `self.value` is initialized. 120 + /// 121 + /// [`stack_pin_init`]: kernel::stack_pin_init 122 + pub struct StackInit<T> { 123 + value: MaybeUninit<T>, 124 + is_init: bool, 125 + } 126 + 127 + impl<T> Drop for StackInit<T> { 128 + #[inline] 129 + fn drop(&mut self) { 130 + if self.is_init { 131 + // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is 132 + // true, `self.value` is initialized. 133 + unsafe { self.value.assume_init_drop() }; 134 + } 135 + } 136 + } 137 + 138 + impl<T> StackInit<T> { 139 + /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this 140 + /// primitive. 141 + /// 142 + /// [`stack_pin_init`]: kernel::stack_pin_init 143 + #[inline] 144 + pub fn uninit() -> Self { 145 + Self { 146 + value: MaybeUninit::uninit(), 147 + is_init: false, 148 + } 149 + } 150 + 151 + /// Initializes the contents and returns the result. 152 + #[inline] 153 + pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mut T>, E> { 154 + // SAFETY: We never move out of `this`. 155 + let this = unsafe { Pin::into_inner_unchecked(self) }; 156 + // The value is currently initialized, so it needs to be dropped before we can reuse 157 + // the memory (this is a safety guarantee of `Pin`). 158 + if this.is_init { 159 + this.is_init = false; 160 + // SAFETY: `this.is_init` was true and therefore `this.value` is initialized. 161 + unsafe { this.value.assume_init_drop() }; 162 + } 163 + // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. 164 + unsafe { init.__pinned_init(this.value.as_mut_ptr())? }; 165 + // INVARIANT: `this.value` is initialized above. 166 + this.is_init = true; 167 + // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. 168 + Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) }) 169 + } 170 + } 171 + 172 + /// When a value of this type is dropped, it drops a `T`. 173 + /// 174 + /// Can be forgotten to prevent the drop. 175 + pub struct DropGuard<T: ?Sized> { 176 + ptr: *mut T, 177 + do_drop: Cell<bool>, 178 + } 179 + 180 + impl<T: ?Sized> DropGuard<T> { 181 + /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped. 182 + /// 183 + /// # Safety 184 + /// 185 + /// `ptr` must be a valid pointer. 186 + /// 187 + /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`: 188 + /// - has not been dropped, 189 + /// - is not accessible by any other means, 190 + /// - will not be dropped by any other means. 191 + #[inline] 192 + pub unsafe fn new(ptr: *mut T) -> Self { 193 + Self { 194 + ptr, 195 + do_drop: Cell::new(true), 196 + } 197 + } 198 + 199 + /// Prevents this guard from dropping the supplied pointer. 200 + /// 201 + /// # Safety 202 + /// 203 + /// This function is unsafe in order to prevent safe code from forgetting this guard. It should 204 + /// only be called by the macros in this module. 205 + #[inline] 206 + pub unsafe fn forget(&self) { 207 + self.do_drop.set(false); 208 + } 209 + } 210 + 211 + impl<T: ?Sized> Drop for DropGuard<T> { 212 + #[inline] 213 + fn drop(&mut self) { 214 + if self.do_drop.get() { 215 + // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function 216 + // ensuring that this operation is safe. 217 + unsafe { ptr::drop_in_place(self.ptr) } 218 + } 219 + } 220 + } 221 + 222 + /// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely 223 + /// created struct. This is needed, because the `drop` function is safe, but should not be called 224 + /// manually. 225 + pub struct OnlyCallFromDrop(()); 226 + 227 + impl OnlyCallFromDrop { 228 + /// # Safety 229 + /// 230 + /// This function should only be called from the [`Drop::drop`] function and only be used to 231 + /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type. 232 + pub unsafe fn new() -> Self { 233 + Self(()) 234 + } 235 + }

+971

rust/kernel/init/macros.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + //! This module provides the macros that actually implement the proc-macros `pin_data` and 4 + //! `pinned_drop`. 5 + //! 6 + //! These macros should never be called directly, since they expect their input to be 7 + //! in a certain format which is internal. Use the proc-macros instead. 8 + //! 9 + //! This architecture has been chosen because the kernel does not yet have access to `syn` which 10 + //! would make matters a lot easier for implementing these as proc-macros. 11 + //! 12 + //! # Macro expansion example 13 + //! 14 + //! This section is intended for readers trying to understand the macros in this module and the 15 + //! `pin_init!` macros from `init.rs`. 16 + //! 17 + //! We will look at the following example: 18 + //! 19 + //! ```rust 20 + //! # use kernel::init::*; 21 + //! #[pin_data] 22 + //! #[repr(C)] 23 + //! struct Bar<T> { 24 + //! #[pin] 25 + //! t: T, 26 + //! pub x: usize, 27 + //! } 28 + //! 29 + //! impl<T> Bar<T> { 30 + //! fn new(t: T) -> impl PinInit<Self> { 31 + //! pin_init!(Self { t, x: 0 }) 32 + //! } 33 + //! } 34 + //! 35 + //! #[pin_data(PinnedDrop)] 36 + //! struct Foo { 37 + //! a: usize, 38 + //! #[pin] 39 + //! b: Bar<u32>, 40 + //! } 41 + //! 42 + //! #[pinned_drop] 43 + //! impl PinnedDrop for Foo { 44 + //! fn drop(self: Pin<&mut Self>) { 45 + //! println!("{self:p} is getting dropped."); 46 + //! } 47 + //! } 48 + //! 49 + //! let a = 42; 50 + //! let initializer = pin_init!(Foo { 51 + //! a, 52 + //! b <- Bar::new(36), 53 + //! }); 54 + //! ``` 55 + //! 56 + //! This example includes the most common and important features of the pin-init API. 57 + //! 58 + //! Below you can find individual section about the different macro invocations. Here are some 59 + //! general things we need to take into account when designing macros: 60 + //! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()` 61 + //! this ensures that the correct item is used, since users could define their own `mod core {}` 62 + //! and then their own `panic!` inside to execute arbitrary code inside of our macro. 63 + //! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied 64 + //! expressions inside of an `unsafe` block in the macro, because this would allow users to do 65 + //! `unsafe` operations without an associated `unsafe` block. 66 + //! 67 + //! ## `#[pin_data]` on `Bar` 68 + //! 69 + //! This macro is used to specify which fields are structurally pinned and which fields are not. It 70 + //! is placed on the struct definition and allows `#[pin]` to be placed on the fields. 71 + //! 72 + //! Here is the definition of `Bar` from our example: 73 + //! 74 + //! ```rust 75 + //! # use kernel::init::*; 76 + //! #[pin_data] 77 + //! #[repr(C)] 78 + //! struct Bar<T> { 79 + //! t: T, 80 + //! pub x: usize, 81 + //! } 82 + //! ``` 83 + //! 84 + //! This expands to the following code: 85 + //! 86 + //! ```rust 87 + //! // Firstly the normal definition of the struct, attributes are preserved: 88 + //! #[repr(C)] 89 + //! struct Bar<T> { 90 + //! t: T, 91 + //! pub x: usize, 92 + //! } 93 + //! // Then an anonymous constant is defined, this is because we do not want any code to access the 94 + //! // types that we define inside: 95 + //! const _: () = { 96 + //! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics, 97 + //! // since we need to implement access functions for each field and thus need to know its 98 + //! // type. 99 + //! struct __ThePinData<T> { 100 + //! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, 101 + //! } 102 + //! // We implement `Copy` for the pin-data struct, since all functions it defines will take 103 + //! // `self` by value. 104 + //! impl<T> ::core::clone::Clone for __ThePinData<T> { 105 + //! fn clone(&self) -> Self { 106 + //! *self 107 + //! } 108 + //! } 109 + //! impl<T> ::core::marker::Copy for __ThePinData<T> {} 110 + //! // For every field of `Bar`, the pin-data struct will define a function with the same name 111 + //! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the 112 + //! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field 113 + //! // (if pinning is structural for the field, then `PinInit` otherwise `Init`). 114 + //! #[allow(dead_code)] 115 + //! impl<T> __ThePinData<T> { 116 + //! unsafe fn t<E>( 117 + //! self, 118 + //! slot: *mut T, 119 + //! init: impl ::kernel::init::Init<T, E>, 120 + //! ) -> ::core::result::Result<(), E> { 121 + //! unsafe { ::kernel::init::Init::__init(init, slot) } 122 + //! } 123 + //! pub unsafe fn x<E>( 124 + //! self, 125 + //! slot: *mut usize, 126 + //! init: impl ::kernel::init::Init<usize, E>, 127 + //! ) -> ::core::result::Result<(), E> { 128 + //! unsafe { ::kernel::init::Init::__init(init, slot) } 129 + //! } 130 + //! } 131 + //! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct 132 + //! // that we constructed beforehand. 133 + //! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> { 134 + //! type PinData = __ThePinData<T>; 135 + //! unsafe fn __pin_data() -> Self::PinData { 136 + //! __ThePinData { 137 + //! __phantom: ::core::marker::PhantomData, 138 + //! } 139 + //! } 140 + //! } 141 + //! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data 142 + //! // struct. This is important to ensure that no user can implement a rouge `__pin_data` 143 + //! // function without using `unsafe`. 144 + //! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> { 145 + //! type Datee = Bar<T>; 146 + //! } 147 + //! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is 148 + //! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned 149 + //! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our 150 + //! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist 151 + //! // for two reasons: 152 + //! // - `__phantom`: every generic must be used, since we cannot really know which generics 153 + //! // are used, we declere all and then use everything here once. 154 + //! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant 155 + //! // over it. The lifetime is needed to work around the limitation that trait bounds must 156 + //! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is 157 + //! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler 158 + //! // into accepting these bounds regardless. 159 + //! #[allow(dead_code)] 160 + //! struct __Unpin<'__pin, T> { 161 + //! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 162 + //! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, 163 + //! } 164 + //! #[doc(hidden)] 165 + //! impl<'__pin, T> 166 + //! ::core::marker::Unpin for Bar<T> where __Unpin<'__pin, T>: ::core::marker::Unpin {} 167 + //! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users 168 + //! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to 169 + //! // UB with only safe code, so we disallow this by giving a trait implementation error using 170 + //! // a direct impl and a blanket implementation. 171 + //! trait MustNotImplDrop {} 172 + //! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do 173 + //! // (normally people want to know if a type has any kind of drop glue at all, here we want 174 + //! // to know if it has any kind of custom drop glue, which is exactly what this bound does). 175 + //! #[allow(drop_bounds)] 176 + //! impl<T: ::core::ops::Drop> MustNotImplDrop for T {} 177 + //! impl<T> MustNotImplDrop for Bar<T> {} 178 + //! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to 179 + //! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed 180 + //! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. 181 + //! #[allow(non_camel_case_types)] 182 + //! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} 183 + //! impl<T: ::kernel::init::PinnedDrop> 184 + //! UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} 185 + //! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {} 186 + //! }; 187 + //! ``` 188 + //! 189 + //! ## `pin_init!` in `impl Bar` 190 + //! 191 + //! This macro creates an pin-initializer for the given struct. It requires that the struct is 192 + //! annotated by `#[pin_data]`. 193 + //! 194 + //! Here is the impl on `Bar` defining the new function: 195 + //! 196 + //! ```rust 197 + //! impl<T> Bar<T> { 198 + //! fn new(t: T) -> impl PinInit<Self> { 199 + //! pin_init!(Self { t, x: 0 }) 200 + //! } 201 + //! } 202 + //! ``` 203 + //! 204 + //! This expands to the following code: 205 + //! 206 + //! ```rust 207 + //! impl<T> Bar<T> { 208 + //! fn new(t: T) -> impl PinInit<Self> { 209 + //! { 210 + //! // We do not want to allow arbitrary returns, so we declare this type as the `Ok` 211 + //! // return type and shadow it later when we insert the arbitrary user code. That way 212 + //! // there will be no possibility of returning without `unsafe`. 213 + //! struct __InitOk; 214 + //! // Get the pin-data type from the initialized type. 215 + //! // - the function is unsafe, hence the unsafe block 216 + //! // - we `use` the `HasPinData` trait in the block, it is only available in that 217 + //! // scope. 218 + //! let data = unsafe { 219 + //! use ::kernel::init::__internal::HasPinData; 220 + //! Self::__pin_data() 221 + //! }; 222 + //! // Use `data` to help with type inference, the closure supplied will have the type 223 + //! // `FnOnce(*mut Self) -> Result<__InitOk, Infallible>`. 224 + //! let init = ::kernel::init::__internal::PinData::make_closure::< 225 + //! _, 226 + //! __InitOk, 227 + //! ::core::convert::Infallible, 228 + //! >(data, move |slot| { 229 + //! { 230 + //! // Shadow the structure so it cannot be used to return early. If a user 231 + //! // tries to write `return Ok(__InitOk)`, then they get a type error, since 232 + //! // that will refer to this struct instead of the one defined above. 233 + //! struct __InitOk; 234 + //! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. 235 + //! unsafe { ::core::ptr::write(&raw mut (*slot).t, t) }; 236 + //! // Since initialization could fail later (not in this case, since the error 237 + //! // type is `Infallible`) we will need to drop this field if it fails. This 238 + //! // `DropGuard` will drop the field when it gets dropped and has not yet 239 + //! // been forgotten. We make a reference to it, so users cannot `mem::forget` 240 + //! // it from the initializer, since the name is the same as the field. 241 + //! let t = &unsafe { 242 + //! ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).t) 243 + //! }; 244 + //! // Expansion of `x: 0,`: 245 + //! // Since this can be an arbitrary expression we cannot place it inside of 246 + //! // the `unsafe` block, so we bind it here. 247 + //! let x = 0; 248 + //! unsafe { ::core::ptr::write(&raw mut (*slot).x, x) }; 249 + //! let x = &unsafe { 250 + //! ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).x) 251 + //! }; 252 + //! 253 + //! // Here we use the type checker to ensuer that every field has been 254 + //! // initialized exactly once, since this is `if false` it will never get 255 + //! // executed, but still type-checked. 256 + //! // Additionally we abuse `slot` to automatically infer the correct type for 257 + //! // the struct. This is also another check that every field is accessible 258 + //! // from this scope. 259 + //! #[allow(unreachable_code, clippy::diverging_sub_expression)] 260 + //! if false { 261 + //! unsafe { 262 + //! ::core::ptr::write( 263 + //! slot, 264 + //! Self { 265 + //! // We only care about typecheck finding every field here, 266 + //! // the expression does not matter, just conjure one using 267 + //! // `panic!()`: 268 + //! t: ::core::panic!(), 269 + //! x: ::core::panic!(), 270 + //! }, 271 + //! ); 272 + //! }; 273 + //! } 274 + //! // Since initialization has successfully completed, we can now forget the 275 + //! // guards. 276 + //! unsafe { ::kernel::init::__internal::DropGuard::forget(t) }; 277 + //! unsafe { ::kernel::init::__internal::DropGuard::forget(x) }; 278 + //! } 279 + //! // We leave the scope above and gain access to the previously shadowed 280 + //! // `__InitOk` that we need to return. 281 + //! Ok(__InitOk) 282 + //! }); 283 + //! // Change the return type of the closure. 284 + //! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { 285 + //! init(slot).map(|__InitOk| ()) 286 + //! }; 287 + //! // Construct the initializer. 288 + //! let init = unsafe { 289 + //! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) 290 + //! }; 291 + //! init 292 + //! } 293 + //! } 294 + //! } 295 + //! ``` 296 + //! 297 + //! ## `#[pin_data]` on `Foo` 298 + //! 299 + //! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the 300 + //! differences/new things in the expansion of the `Foo` definition: 301 + //! 302 + //! ```rust 303 + //! #[pin_data(PinnedDrop)] 304 + //! struct Foo { 305 + //! a: usize, 306 + //! #[pin] 307 + //! b: Bar<u32>, 308 + //! } 309 + //! ``` 310 + //! 311 + //! This expands to the following code: 312 + //! 313 + //! ```rust 314 + //! struct Foo { 315 + //! a: usize, 316 + //! b: Bar<u32>, 317 + //! } 318 + //! const _: () = { 319 + //! struct __ThePinData { 320 + //! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, 321 + //! } 322 + //! impl ::core::clone::Clone for __ThePinData { 323 + //! fn clone(&self) -> Self { 324 + //! *self 325 + //! } 326 + //! } 327 + //! impl ::core::marker::Copy for __ThePinData {} 328 + //! #[allow(dead_code)] 329 + //! impl __ThePinData { 330 + //! unsafe fn b<E>( 331 + //! self, 332 + //! slot: *mut Bar<u32>, 333 + //! // Note that this is `PinInit` instead of `Init`, this is because `b` is 334 + //! // structurally pinned, as marked by the `#[pin]` attribute. 335 + //! init: impl ::kernel::init::PinInit<Bar<u32>, E>, 336 + //! ) -> ::core::result::Result<(), E> { 337 + //! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } 338 + //! } 339 + //! unsafe fn a<E>( 340 + //! self, 341 + //! slot: *mut usize, 342 + //! init: impl ::kernel::init::Init<usize, E>, 343 + //! ) -> ::core::result::Result<(), E> { 344 + //! unsafe { ::kernel::init::Init::__init(init, slot) } 345 + //! } 346 + //! } 347 + //! unsafe impl ::kernel::init::__internal::HasPinData for Foo { 348 + //! type PinData = __ThePinData; 349 + //! unsafe fn __pin_data() -> Self::PinData { 350 + //! __ThePinData { 351 + //! __phantom: ::core::marker::PhantomData, 352 + //! } 353 + //! } 354 + //! } 355 + //! unsafe impl ::kernel::init::__internal::PinData for __ThePinData { 356 + //! type Datee = Foo; 357 + //! } 358 + //! #[allow(dead_code)] 359 + //! struct __Unpin<'__pin> { 360 + //! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 361 + //! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, 362 + //! // Since this field is `#[pin]`, it is listed here. 363 + //! b: Bar<u32>, 364 + //! } 365 + //! #[doc(hidden)] 366 + //! impl<'__pin> ::core::marker::Unpin for Foo where __Unpin<'__pin>: ::core::marker::Unpin {} 367 + //! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to 368 + //! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like 369 + //! // before, instead we implement it here and delegate to `PinnedDrop`. 370 + //! impl ::core::ops::Drop for Foo { 371 + //! fn drop(&mut self) { 372 + //! // Since we are getting dropped, no one else has a reference to `self` and thus we 373 + //! // can assume that we never move. 374 + //! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; 375 + //! // Create the unsafe token that proves that we are inside of a destructor, this 376 + //! // type is only allowed to be created in a destructor. 377 + //! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() }; 378 + //! ::kernel::init::PinnedDrop::drop(pinned, token); 379 + //! } 380 + //! } 381 + //! }; 382 + //! ``` 383 + //! 384 + //! ## `#[pinned_drop]` on `impl PinnedDrop for Foo` 385 + //! 386 + //! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an 387 + //! extra parameter that should not be used at all. The macro hides that parameter. 388 + //! 389 + //! Here is the `PinnedDrop` impl for `Foo`: 390 + //! 391 + //! ```rust 392 + //! #[pinned_drop] 393 + //! impl PinnedDrop for Foo { 394 + //! fn drop(self: Pin<&mut Self>) { 395 + //! println!("{self:p} is getting dropped."); 396 + //! } 397 + //! } 398 + //! ``` 399 + //! 400 + //! This expands to the following code: 401 + //! 402 + //! ```rust 403 + //! // `unsafe`, full path and the token parameter are added, everything else stays the same. 404 + //! unsafe impl ::kernel::init::PinnedDrop for Foo { 405 + //! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) { 406 + //! println!("{self:p} is getting dropped."); 407 + //! } 408 + //! } 409 + //! ``` 410 + //! 411 + //! ## `pin_init!` on `Foo` 412 + //! 413 + //! Since we already took a look at `pin_init!` on `Bar`, this section will only explain the 414 + //! differences/new things in the expansion of `pin_init!` on `Foo`: 415 + //! 416 + //! ```rust 417 + //! let a = 42; 418 + //! let initializer = pin_init!(Foo { 419 + //! a, 420 + //! b <- Bar::new(36), 421 + //! }); 422 + //! ``` 423 + //! 424 + //! This expands to the following code: 425 + //! 426 + //! ```rust 427 + //! let a = 42; 428 + //! let initializer = { 429 + //! struct __InitOk; 430 + //! let data = unsafe { 431 + //! use ::kernel::init::__internal::HasPinData; 432 + //! Foo::__pin_data() 433 + //! }; 434 + //! let init = ::kernel::init::__internal::PinData::make_closure::< 435 + //! _, 436 + //! __InitOk, 437 + //! ::core::convert::Infallible, 438 + //! >(data, move |slot| { 439 + //! { 440 + //! struct __InitOk; 441 + //! unsafe { ::core::ptr::write(&raw mut (*slot).a, a) }; 442 + //! let a = &unsafe { ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).a) }; 443 + //! let b = Bar::new(36); 444 + //! // Here we use `data` to access the correct field and require that `b` is of type 445 + //! // `PinInit<Bar<u32>, Infallible>`. 446 + //! unsafe { data.b(&raw mut (*slot).b, b)? }; 447 + //! let b = &unsafe { ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).b) }; 448 + //! 449 + //! #[allow(unreachable_code, clippy::diverging_sub_expression)] 450 + //! if false { 451 + //! unsafe { 452 + //! ::core::ptr::write( 453 + //! slot, 454 + //! Foo { 455 + //! a: ::core::panic!(), 456 + //! b: ::core::panic!(), 457 + //! }, 458 + //! ); 459 + //! }; 460 + //! } 461 + //! unsafe { ::kernel::init::__internal::DropGuard::forget(a) }; 462 + //! unsafe { ::kernel::init::__internal::DropGuard::forget(b) }; 463 + //! } 464 + //! Ok(__InitOk) 465 + //! }); 466 + //! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { 467 + //! init(slot).map(|__InitOk| ()) 468 + //! }; 469 + //! let init = unsafe { 470 + //! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) 471 + //! }; 472 + //! init 473 + //! }; 474 + //! ``` 475 + 476 + /// Creates a `unsafe impl<...> PinnedDrop for $type` block. 477 + /// 478 + /// See [`PinnedDrop`] for more information. 479 + #[doc(hidden)] 480 + #[macro_export] 481 + macro_rules! __pinned_drop { 482 + ( 483 + @impl_sig($($impl_sig:tt)*), 484 + @impl_body( 485 + $(#[$($attr:tt)*])* 486 + fn drop($($sig:tt)*) { 487 + $($inner:tt)* 488 + } 489 + ), 490 + ) => { 491 + unsafe $($impl_sig)* { 492 + // Inherit all attributes and the type/ident tokens for the signature. 493 + $(#[$($attr)*])* 494 + fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) { 495 + $($inner)* 496 + } 497 + } 498 + } 499 + } 500 + 501 + /// This macro first parses the struct definition such that it separates pinned and not pinned 502 + /// fields. Afterwards it declares the struct and implement the `PinData` trait safely. 503 + #[doc(hidden)] 504 + #[macro_export] 505 + macro_rules! __pin_data { 506 + // Proc-macro entry point, this is supplied by the proc-macro pre-parsing. 507 + (parse_input: 508 + @args($($pinned_drop:ident)?), 509 + @sig( 510 + $(#[$($struct_attr:tt)*])* 511 + $vis:vis struct $name:ident 512 + $(where $($whr:tt)*)? 513 + ), 514 + @impl_generics($($impl_generics:tt)*), 515 + @ty_generics($($ty_generics:tt)*), 516 + @body({ $($fields:tt)* }), 517 + ) => { 518 + // We now use token munching to iterate through all of the fields. While doing this we 519 + // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user 520 + // wants these to be structurally pinned. The rest of the fields are the 521 + // 'not pinned fields'. Additionally we collect all fields, since we need them in the right 522 + // order to declare the struct. 523 + // 524 + // In this call we also put some explaining comments for the parameters. 525 + $crate::__pin_data!(find_pinned_fields: 526 + // Attributes on the struct itself, these will just be propagated to be put onto the 527 + // struct definition. 528 + @struct_attrs($(#[$($struct_attr)*])*), 529 + // The visibility of the struct. 530 + @vis($vis), 531 + // The name of the struct. 532 + @name($name), 533 + // The 'impl generics', the generics that will need to be specified on the struct inside 534 + // of an `impl<$ty_generics>` block. 535 + @impl_generics($($impl_generics)*), 536 + // The 'ty generics', the generics that will need to be specified on the impl blocks. 537 + @ty_generics($($ty_generics)*), 538 + // The where clause of any impl block and the declaration. 539 + @where($($($whr)*)?), 540 + // The remaining fields tokens that need to be processed. 541 + // We add a `,` at the end to ensure correct parsing. 542 + @fields_munch($($fields)* ,), 543 + // The pinned fields. 544 + @pinned(), 545 + // The not pinned fields. 546 + @not_pinned(), 547 + // All fields. 548 + @fields(), 549 + // The accumulator containing all attributes already parsed. 550 + @accum(), 551 + // Contains `yes` or `` to indicate if `#[pin]` was found on the current field. 552 + @is_pinned(), 553 + // The proc-macro argument, this should be `PinnedDrop` or ``. 554 + @pinned_drop($($pinned_drop)?), 555 + ); 556 + }; 557 + (find_pinned_fields: 558 + @struct_attrs($($struct_attrs:tt)*), 559 + @vis($vis:vis), 560 + @name($name:ident), 561 + @impl_generics($($impl_generics:tt)*), 562 + @ty_generics($($ty_generics:tt)*), 563 + @where($($whr:tt)*), 564 + // We found a PhantomPinned field, this should generally be pinned! 565 + @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), 566 + @pinned($($pinned:tt)*), 567 + @not_pinned($($not_pinned:tt)*), 568 + @fields($($fields:tt)*), 569 + @accum($($accum:tt)*), 570 + // This field is not pinned. 571 + @is_pinned(), 572 + @pinned_drop($($pinned_drop:ident)?), 573 + ) => { 574 + ::core::compile_error!(concat!( 575 + "The field `", 576 + stringify!($field), 577 + "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.", 578 + )); 579 + $crate::__pin_data!(find_pinned_fields: 580 + @struct_attrs($($struct_attrs)*), 581 + @vis($vis), 582 + @name($name), 583 + @impl_generics($($impl_generics)*), 584 + @ty_generics($($ty_generics)*), 585 + @where($($whr)*), 586 + @fields_munch($($rest)*), 587 + @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), 588 + @not_pinned($($not_pinned)*), 589 + @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,), 590 + @accum(), 591 + @is_pinned(), 592 + @pinned_drop($($pinned_drop)?), 593 + ); 594 + }; 595 + (find_pinned_fields: 596 + @struct_attrs($($struct_attrs:tt)*), 597 + @vis($vis:vis), 598 + @name($name:ident), 599 + @impl_generics($($impl_generics:tt)*), 600 + @ty_generics($($ty_generics:tt)*), 601 + @where($($whr:tt)*), 602 + // We reached the field declaration. 603 + @fields_munch($field:ident : $type:ty, $($rest:tt)*), 604 + @pinned($($pinned:tt)*), 605 + @not_pinned($($not_pinned:tt)*), 606 + @fields($($fields:tt)*), 607 + @accum($($accum:tt)*), 608 + // This field is pinned. 609 + @is_pinned(yes), 610 + @pinned_drop($($pinned_drop:ident)?), 611 + ) => { 612 + $crate::__pin_data!(find_pinned_fields: 613 + @struct_attrs($($struct_attrs)*), 614 + @vis($vis), 615 + @name($name), 616 + @impl_generics($($impl_generics)*), 617 + @ty_generics($($ty_generics)*), 618 + @where($($whr)*), 619 + @fields_munch($($rest)*), 620 + @pinned($($pinned)* $($accum)* $field: $type,), 621 + @not_pinned($($not_pinned)*), 622 + @fields($($fields)* $($accum)* $field: $type,), 623 + @accum(), 624 + @is_pinned(), 625 + @pinned_drop($($pinned_drop)?), 626 + ); 627 + }; 628 + (find_pinned_fields: 629 + @struct_attrs($($struct_attrs:tt)*), 630 + @vis($vis:vis), 631 + @name($name:ident), 632 + @impl_generics($($impl_generics:tt)*), 633 + @ty_generics($($ty_generics:tt)*), 634 + @where($($whr:tt)*), 635 + // We reached the field declaration. 636 + @fields_munch($field:ident : $type:ty, $($rest:tt)*), 637 + @pinned($($pinned:tt)*), 638 + @not_pinned($($not_pinned:tt)*), 639 + @fields($($fields:tt)*), 640 + @accum($($accum:tt)*), 641 + // This field is not pinned. 642 + @is_pinned(), 643 + @pinned_drop($($pinned_drop:ident)?), 644 + ) => { 645 + $crate::__pin_data!(find_pinned_fields: 646 + @struct_attrs($($struct_attrs)*), 647 + @vis($vis), 648 + @name($name), 649 + @impl_generics($($impl_generics)*), 650 + @ty_generics($($ty_generics)*), 651 + @where($($whr)*), 652 + @fields_munch($($rest)*), 653 + @pinned($($pinned)*), 654 + @not_pinned($($not_pinned)* $($accum)* $field: $type,), 655 + @fields($($fields)* $($accum)* $field: $type,), 656 + @accum(), 657 + @is_pinned(), 658 + @pinned_drop($($pinned_drop)?), 659 + ); 660 + }; 661 + (find_pinned_fields: 662 + @struct_attrs($($struct_attrs:tt)*), 663 + @vis($vis:vis), 664 + @name($name:ident), 665 + @impl_generics($($impl_generics:tt)*), 666 + @ty_generics($($ty_generics:tt)*), 667 + @where($($whr:tt)*), 668 + // We found the `#[pin]` attr. 669 + @fields_munch(#[pin] $($rest:tt)*), 670 + @pinned($($pinned:tt)*), 671 + @not_pinned($($not_pinned:tt)*), 672 + @fields($($fields:tt)*), 673 + @accum($($accum:tt)*), 674 + @is_pinned($($is_pinned:ident)?), 675 + @pinned_drop($($pinned_drop:ident)?), 676 + ) => { 677 + $crate::__pin_data!(find_pinned_fields: 678 + @struct_attrs($($struct_attrs)*), 679 + @vis($vis), 680 + @name($name), 681 + @impl_generics($($impl_generics)*), 682 + @ty_generics($($ty_generics)*), 683 + @where($($whr)*), 684 + @fields_munch($($rest)*), 685 + // We do not include `#[pin]` in the list of attributes, since it is not actually an 686 + // attribute that is defined somewhere. 687 + @pinned($($pinned)*), 688 + @not_pinned($($not_pinned)*), 689 + @fields($($fields)*), 690 + @accum($($accum)*), 691 + // Set this to `yes`. 692 + @is_pinned(yes), 693 + @pinned_drop($($pinned_drop)?), 694 + ); 695 + }; 696 + (find_pinned_fields: 697 + @struct_attrs($($struct_attrs:tt)*), 698 + @vis($vis:vis), 699 + @name($name:ident), 700 + @impl_generics($($impl_generics:tt)*), 701 + @ty_generics($($ty_generics:tt)*), 702 + @where($($whr:tt)*), 703 + // We reached the field declaration with visibility, for simplicity we only munch the 704 + // visibility and put it into `$accum`. 705 + @fields_munch($fvis:vis $field:ident $($rest:tt)*), 706 + @pinned($($pinned:tt)*), 707 + @not_pinned($($not_pinned:tt)*), 708 + @fields($($fields:tt)*), 709 + @accum($($accum:tt)*), 710 + @is_pinned($($is_pinned:ident)?), 711 + @pinned_drop($($pinned_drop:ident)?), 712 + ) => { 713 + $crate::__pin_data!(find_pinned_fields: 714 + @struct_attrs($($struct_attrs)*), 715 + @vis($vis), 716 + @name($name), 717 + @impl_generics($($impl_generics)*), 718 + @ty_generics($($ty_generics)*), 719 + @where($($whr)*), 720 + @fields_munch($field $($rest)*), 721 + @pinned($($pinned)*), 722 + @not_pinned($($not_pinned)*), 723 + @fields($($fields)*), 724 + @accum($($accum)* $fvis), 725 + @is_pinned($($is_pinned)?), 726 + @pinned_drop($($pinned_drop)?), 727 + ); 728 + }; 729 + (find_pinned_fields: 730 + @struct_attrs($($struct_attrs:tt)*), 731 + @vis($vis:vis), 732 + @name($name:ident), 733 + @impl_generics($($impl_generics:tt)*), 734 + @ty_generics($($ty_generics:tt)*), 735 + @where($($whr:tt)*), 736 + // Some other attribute, just put it into `$accum`. 737 + @fields_munch(#[$($attr:tt)*] $($rest:tt)*), 738 + @pinned($($pinned:tt)*), 739 + @not_pinned($($not_pinned:tt)*), 740 + @fields($($fields:tt)*), 741 + @accum($($accum:tt)*), 742 + @is_pinned($($is_pinned:ident)?), 743 + @pinned_drop($($pinned_drop:ident)?), 744 + ) => { 745 + $crate::__pin_data!(find_pinned_fields: 746 + @struct_attrs($($struct_attrs)*), 747 + @vis($vis), 748 + @name($name), 749 + @impl_generics($($impl_generics)*), 750 + @ty_generics($($ty_generics)*), 751 + @where($($whr)*), 752 + @fields_munch($($rest)*), 753 + @pinned($($pinned)*), 754 + @not_pinned($($not_pinned)*), 755 + @fields($($fields)*), 756 + @accum($($accum)* #[$($attr)*]), 757 + @is_pinned($($is_pinned)?), 758 + @pinned_drop($($pinned_drop)?), 759 + ); 760 + }; 761 + (find_pinned_fields: 762 + @struct_attrs($($struct_attrs:tt)*), 763 + @vis($vis:vis), 764 + @name($name:ident), 765 + @impl_generics($($impl_generics:tt)*), 766 + @ty_generics($($ty_generics:tt)*), 767 + @where($($whr:tt)*), 768 + // We reached the end of the fields, plus an optional additional comma, since we added one 769 + // before and the user is also allowed to put a trailing comma. 770 + @fields_munch($(,)?), 771 + @pinned($($pinned:tt)*), 772 + @not_pinned($($not_pinned:tt)*), 773 + @fields($($fields:tt)*), 774 + @accum(), 775 + @is_pinned(), 776 + @pinned_drop($($pinned_drop:ident)?), 777 + ) => { 778 + // Declare the struct with all fields in the correct order. 779 + $($struct_attrs)* 780 + $vis struct $name <$($impl_generics)*> 781 + where $($whr)* 782 + { 783 + $($fields)* 784 + } 785 + 786 + // We put the rest into this const item, because it then will not be accessible to anything 787 + // outside. 788 + const _: () = { 789 + // We declare this struct which will host all of the projection function for our type. 790 + // it will be invariant over all generic parameters which are inherited from the 791 + // struct. 792 + $vis struct __ThePinData<$($impl_generics)*> 793 + where $($whr)* 794 + { 795 + __phantom: ::core::marker::PhantomData< 796 + fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> 797 + >, 798 + } 799 + 800 + impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*> 801 + where $($whr)* 802 + { 803 + fn clone(&self) -> Self { *self } 804 + } 805 + 806 + impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*> 807 + where $($whr)* 808 + {} 809 + 810 + // Make all projection functions. 811 + $crate::__pin_data!(make_pin_data: 812 + @pin_data(__ThePinData), 813 + @impl_generics($($impl_generics)*), 814 + @ty_generics($($ty_generics)*), 815 + @where($($whr)*), 816 + @pinned($($pinned)*), 817 + @not_pinned($($not_pinned)*), 818 + ); 819 + 820 + // SAFETY: We have added the correct projection functions above to `__ThePinData` and 821 + // we also use the least restrictive generics possible. 822 + unsafe impl<$($impl_generics)*> 823 + $crate::init::__internal::HasPinData for $name<$($ty_generics)*> 824 + where $($whr)* 825 + { 826 + type PinData = __ThePinData<$($ty_generics)*>; 827 + 828 + unsafe fn __pin_data() -> Self::PinData { 829 + __ThePinData { __phantom: ::core::marker::PhantomData } 830 + } 831 + } 832 + 833 + unsafe impl<$($impl_generics)*> 834 + $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*> 835 + where $($whr)* 836 + { 837 + type Datee = $name<$($ty_generics)*>; 838 + } 839 + 840 + // This struct will be used for the unpin analysis. Since only structurally pinned 841 + // fields are relevant whether the struct should implement `Unpin`. 842 + #[allow(dead_code)] 843 + struct __Unpin <'__pin, $($impl_generics)*> 844 + where $($whr)* 845 + { 846 + __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 847 + __phantom: ::core::marker::PhantomData< 848 + fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> 849 + >, 850 + // Only the pinned fields. 851 + $($pinned)* 852 + } 853 + 854 + #[doc(hidden)] 855 + impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*> 856 + where 857 + __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin, 858 + $($whr)* 859 + {} 860 + 861 + // We need to disallow normal `Drop` implementation, the exact behavior depends on 862 + // whether `PinnedDrop` was specified as the parameter. 863 + $crate::__pin_data!(drop_prevention: 864 + @name($name), 865 + @impl_generics($($impl_generics)*), 866 + @ty_generics($($ty_generics)*), 867 + @where($($whr)*), 868 + @pinned_drop($($pinned_drop)?), 869 + ); 870 + }; 871 + }; 872 + // When no `PinnedDrop` was specified, then we have to prevent implementing drop. 873 + (drop_prevention: 874 + @name($name:ident), 875 + @impl_generics($($impl_generics:tt)*), 876 + @ty_generics($($ty_generics:tt)*), 877 + @where($($whr:tt)*), 878 + @pinned_drop(), 879 + ) => { 880 + // We prevent this by creating a trait that will be implemented for all types implementing 881 + // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, 882 + // if it also implements `Drop` 883 + trait MustNotImplDrop {} 884 + #[allow(drop_bounds)] 885 + impl<T: ::core::ops::Drop> MustNotImplDrop for T {} 886 + impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*> 887 + where $($whr)* {} 888 + // We also take care to prevent users from writing a useless `PinnedDrop` implementation. 889 + // They might implement `PinnedDrop` correctly for the struct, but forget to give 890 + // `PinnedDrop` as the parameter to `#[pin_data]`. 891 + #[allow(non_camel_case_types)] 892 + trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} 893 + impl<T: $crate::init::PinnedDrop> 894 + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} 895 + impl<$($impl_generics)*> 896 + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*> 897 + where $($whr)* {} 898 + }; 899 + // When `PinnedDrop` was specified we just implement `Drop` and delegate. 900 + (drop_prevention: 901 + @name($name:ident), 902 + @impl_generics($($impl_generics:tt)*), 903 + @ty_generics($($ty_generics:tt)*), 904 + @where($($whr:tt)*), 905 + @pinned_drop(PinnedDrop), 906 + ) => { 907 + impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*> 908 + where $($whr)* 909 + { 910 + fn drop(&mut self) { 911 + // SAFETY: Since this is a destructor, `self` will not move after this function 912 + // terminates, since it is inaccessible. 913 + let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; 914 + // SAFETY: Since this is a drop function, we can create this token to call the 915 + // pinned destructor of this type. 916 + let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() }; 917 + $crate::init::PinnedDrop::drop(pinned, token); 918 + } 919 + } 920 + }; 921 + // If some other parameter was specified, we emit a readable error. 922 + (drop_prevention: 923 + @name($name:ident), 924 + @impl_generics($($impl_generics:tt)*), 925 + @ty_generics($($ty_generics:tt)*), 926 + @where($($whr:tt)*), 927 + @pinned_drop($($rest:tt)*), 928 + ) => { 929 + compile_error!( 930 + "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.", 931 + stringify!($($rest)*), 932 + ); 933 + }; 934 + (make_pin_data: 935 + @pin_data($pin_data:ident), 936 + @impl_generics($($impl_generics:tt)*), 937 + @ty_generics($($ty_generics:tt)*), 938 + @where($($whr:tt)*), 939 + @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?), 940 + @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?), 941 + ) => { 942 + // For every field, we create a projection function according to its projection type. If a 943 + // field is structurally pinned, then it must be initialized via `PinInit`, if it is not 944 + // structurally pinned, then it can be initialized via `Init`. 945 + // 946 + // The functions are `unsafe` to prevent accidentally calling them. 947 + #[allow(dead_code)] 948 + impl<$($impl_generics)*> $pin_data<$($ty_generics)*> 949 + where $($whr)* 950 + { 951 + $( 952 + $pvis unsafe fn $p_field<E>( 953 + self, 954 + slot: *mut $p_type, 955 + init: impl $crate::init::PinInit<$p_type, E>, 956 + ) -> ::core::result::Result<(), E> { 957 + unsafe { $crate::init::PinInit::__pinned_init(init, slot) } 958 + } 959 + )* 960 + $( 961 + $fvis unsafe fn $field<E>( 962 + self, 963 + slot: *mut $type, 964 + init: impl $crate::init::Init<$type, E>, 965 + ) -> ::core::result::Result<(), E> { 966 + unsafe { $crate::init::Init::__init(init, slot) } 967 + } 968 + )* 969 + } 970 + }; 971 + }

+72

rust/kernel/ioctl.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + //! ioctl() number definitions 4 + //! 5 + //! C header: [`include/asm-generic/ioctl.h`](../../../../include/asm-generic/ioctl.h) 6 + 7 + #![allow(non_snake_case)] 8 + 9 + use crate::build_assert; 10 + 11 + /// Build an ioctl number, analogous to the C macro of the same name. 12 + #[inline(always)] 13 + const fn _IOC(dir: u32, ty: u32, nr: u32, size: usize) -> u32 { 14 + build_assert!(dir <= uapi::_IOC_DIRMASK); 15 + build_assert!(ty <= uapi::_IOC_TYPEMASK); 16 + build_assert!(nr <= uapi::_IOC_NRMASK); 17 + build_assert!(size <= (uapi::_IOC_SIZEMASK as usize)); 18 + 19 + (dir << uapi::_IOC_DIRSHIFT) 20 + | (ty << uapi::_IOC_TYPESHIFT) 21 + | (nr << uapi::_IOC_NRSHIFT) 22 + | ((size as u32) << uapi::_IOC_SIZESHIFT) 23 + } 24 + 25 + /// Build an ioctl number for an argumentless ioctl. 26 + #[inline(always)] 27 + pub const fn _IO(ty: u32, nr: u32) -> u32 { 28 + _IOC(uapi::_IOC_NONE, ty, nr, 0) 29 + } 30 + 31 + /// Build an ioctl number for an read-only ioctl. 32 + #[inline(always)] 33 + pub const fn _IOR<T>(ty: u32, nr: u32) -> u32 { 34 + _IOC(uapi::_IOC_READ, ty, nr, core::mem::size_of::<T>()) 35 + } 36 + 37 + /// Build an ioctl number for an write-only ioctl. 38 + #[inline(always)] 39 + pub const fn _IOW<T>(ty: u32, nr: u32) -> u32 { 40 + _IOC(uapi::_IOC_WRITE, ty, nr, core::mem::size_of::<T>()) 41 + } 42 + 43 + /// Build an ioctl number for a read-write ioctl. 44 + #[inline(always)] 45 + pub const fn _IOWR<T>(ty: u32, nr: u32) -> u32 { 46 + _IOC( 47 + uapi::_IOC_READ | uapi::_IOC_WRITE, 48 + ty, 49 + nr, 50 + core::mem::size_of::<T>(), 51 + ) 52 + } 53 + 54 + /// Get the ioctl direction from an ioctl number. 55 + pub const fn _IOC_DIR(nr: u32) -> u32 { 56 + (nr >> uapi::_IOC_DIRSHIFT) & uapi::_IOC_DIRMASK 57 + } 58 + 59 + /// Get the ioctl type from an ioctl number. 60 + pub const fn _IOC_TYPE(nr: u32) -> u32 { 61 + (nr >> uapi::_IOC_TYPESHIFT) & uapi::_IOC_TYPEMASK 62 + } 63 + 64 + /// Get the ioctl number from an ioctl number. 65 + pub const fn _IOC_NR(nr: u32) -> u32 { 66 + (nr >> uapi::_IOC_NRSHIFT) & uapi::_IOC_NRMASK 67 + } 68 + 69 + /// Get the ioctl size from an ioctl number. 70 + pub const fn _IOC_SIZE(nr: u32) -> usize { 71 + ((nr >> uapi::_IOC_SIZESHIFT) & uapi::_IOC_SIZEMASK) as usize 72 + }

+10

rust/kernel/lib.rs

··· 16 16 #![feature(coerce_unsized)] 17 17 #![feature(core_ffi_c)] 18 18 #![feature(dispatch_from_dyn)] 19 + #![feature(explicit_generic_args_with_impl_trait)] 19 20 #![feature(generic_associated_types)] 21 + #![feature(new_uninit)] 22 + #![feature(pin_macro)] 20 23 #![feature(receiver_trait)] 21 24 #![feature(unsize)] 22 25 ··· 28 25 #[cfg(not(CONFIG_RUST))] 29 26 compile_error!("Missing kernel configuration for conditional compilation"); 30 27 28 + // Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate). 29 + extern crate self as kernel; 30 + 31 31 #[cfg(not(test))] 32 32 #[cfg(not(testlib))] 33 33 mod allocator; 34 34 mod build_assert; 35 35 pub mod error; 36 + pub mod init; 37 + pub mod ioctl; 36 38 pub mod prelude; 37 39 pub mod print; 38 40 mod static_assert; ··· 45 37 pub mod std_vendor; 46 38 pub mod str; 47 39 pub mod sync; 40 + pub mod task; 48 41 pub mod types; 49 42 50 43 #[doc(hidden)] 51 44 pub use bindings; 52 45 pub use macros; 46 + pub use uapi; 53 47 54 48 #[doc(hidden)] 55 49 pub use build_error::build_error;

+7 -1

rust/kernel/prelude.rs

··· 18 18 pub use alloc::{boxed::Box, vec::Vec}; 19 19 20 20 #[doc(no_inline)] 21 - pub use macros::{module, vtable}; 21 + pub use macros::{module, pin_data, pinned_drop, vtable}; 22 22 23 23 pub use super::build_assert; 24 24 ··· 27 27 pub use super::dbg; 28 28 pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; 29 29 30 + pub use super::{init, pin_init, try_init, try_pin_init}; 31 + 30 32 pub use super::static_assert; 31 33 32 34 pub use super::error::{code::*, Error, Result}; 33 35 34 36 pub use super::{str::CStr, ThisModule}; 37 + 38 + pub use super::init::{InPlaceInit, Init, PinInit}; 39 + 40 + pub use super::current;

+50

rust/kernel/sync.rs

··· 5 5 //! This module contains the kernel APIs related to synchronisation that have been ported or 6 6 //! wrapped for usage by Rust code in the kernel. 7 7 8 + use crate::types::Opaque; 9 + 8 10 mod arc; 11 + mod condvar; 12 + pub mod lock; 13 + mod locked_by; 9 14 10 15 pub use arc::{Arc, ArcBorrow, UniqueArc}; 16 + pub use condvar::CondVar; 17 + pub use lock::{mutex::Mutex, spinlock::SpinLock}; 18 + pub use locked_by::LockedBy; 19 + 20 + /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. 21 + #[repr(transparent)] 22 + pub struct LockClassKey(Opaque<bindings::lock_class_key>); 23 + 24 + // SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and 25 + // provides its own synchronization. 26 + unsafe impl Sync for LockClassKey {} 27 + 28 + impl LockClassKey { 29 + /// Creates a new lock class key. 30 + pub const fn new() -> Self { 31 + Self(Opaque::uninit()) 32 + } 33 + 34 + pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { 35 + self.0.get() 36 + } 37 + } 38 + 39 + /// Defines a new static lock class and returns a pointer to it. 40 + #[doc(hidden)] 41 + #[macro_export] 42 + macro_rules! static_lock_class { 43 + () => {{ 44 + static CLASS: $crate::sync::LockClassKey = $crate::sync::LockClassKey::new(); 45 + &CLASS 46 + }}; 47 + } 48 + 49 + /// Returns the given string, if one is provided, otherwise generates one based on the source code 50 + /// location. 51 + #[doc(hidden)] 52 + #[macro_export] 53 + macro_rules! optional_name { 54 + () => { 55 + $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())) 56 + }; 57 + ($name:literal) => { 58 + $crate::c_str!($name) 59 + }; 60 + }

+102 -6

rust/kernel/sync/arc.rs

··· 17 17 18 18 use crate::{ 19 19 bindings, 20 - error::Result, 20 + error::{self, Error}, 21 + init::{self, InPlaceInit, Init, PinInit}, 22 + try_init, 21 23 types::{ForeignOwnable, Opaque}, 22 24 }; 23 25 use alloc::boxed::Box; 24 26 use core::{ 27 + alloc::AllocError, 28 + fmt, 25 29 marker::{PhantomData, Unsize}, 26 30 mem::{ManuallyDrop, MaybeUninit}, 27 31 ops::{Deref, DerefMut}, 28 32 pin::Pin, 29 33 ptr::NonNull, 30 34 }; 35 + use macros::pin_data; 36 + 37 + mod std_vendor; 31 38 32 39 /// A reference-counted pointer to an instance of `T`. 33 40 /// ··· 127 120 _p: PhantomData<ArcInner<T>>, 128 121 } 129 122 123 + #[pin_data] 130 124 #[repr(C)] 131 125 struct ArcInner<T: ?Sized> { 132 126 refcount: Opaque<bindings::refcount_t>, ··· 157 149 158 150 impl<T> Arc<T> { 159 151 /// Constructs a new reference counted instance of `T`. 160 - pub fn try_new(contents: T) -> Result<Self> { 152 + pub fn try_new(contents: T) -> Result<Self, AllocError> { 161 153 // INVARIANT: The refcount is initialised to a non-zero value. 162 154 let value = ArcInner { 163 155 // SAFETY: There are no safety requirements for this FFI call. ··· 170 162 // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new 171 163 // `Arc` object. 172 164 Ok(unsafe { Self::from_inner(Box::leak(inner).into()) }) 165 + } 166 + 167 + /// Use the given initializer to in-place initialize a `T`. 168 + /// 169 + /// If `T: !Unpin` it will not be able to move afterwards. 170 + #[inline] 171 + pub fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Self> 172 + where 173 + Error: From<E>, 174 + { 175 + UniqueArc::pin_init(init).map(|u| u.into()) 176 + } 177 + 178 + /// Use the given initializer to in-place initialize a `T`. 179 + /// 180 + /// This is equivalent to [`pin_init`], since an [`Arc`] is always pinned. 181 + #[inline] 182 + pub fn init<E>(init: impl Init<T, E>) -> error::Result<Self> 183 + where 184 + Error: From<E>, 185 + { 186 + UniqueArc::init(init).map(|u| u.into()) 173 187 } 174 188 } 175 189 ··· 499 469 500 470 impl<T> UniqueArc<T> { 501 471 /// Tries to allocate a new [`UniqueArc`] instance. 502 - pub fn try_new(value: T) -> Result<Self> { 472 + pub fn try_new(value: T) -> Result<Self, AllocError> { 503 473 Ok(Self { 504 474 // INVARIANT: The newly-created object has a ref-count of 1. 505 475 inner: Arc::try_new(value)?, ··· 507 477 } 508 478 509 479 /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. 510 - pub fn try_new_uninit() -> Result<UniqueArc<MaybeUninit<T>>> { 511 - Ok(UniqueArc::<MaybeUninit<T>> { 480 + pub fn try_new_uninit() -> Result<UniqueArc<MaybeUninit<T>>, AllocError> { 481 + // INVARIANT: The refcount is initialised to a non-zero value. 482 + let inner = Box::try_init::<AllocError>(try_init!(ArcInner { 483 + // SAFETY: There are no safety requirements for this FFI call. 484 + refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), 485 + data <- init::uninit::<T, AllocError>(), 486 + }? AllocError))?; 487 + Ok(UniqueArc { 512 488 // INVARIANT: The newly-created object has a ref-count of 1. 513 - inner: Arc::try_new(MaybeUninit::uninit())?, 489 + // SAFETY: The pointer from the `Box` is valid. 490 + inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, 514 491 }) 515 492 } 516 493 } ··· 526 489 /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it. 527 490 pub fn write(mut self, value: T) -> UniqueArc<T> { 528 491 self.deref_mut().write(value); 492 + // SAFETY: We just wrote the value to be initialized. 493 + unsafe { self.assume_init() } 494 + } 495 + 496 + /// Unsafely assume that `self` is initialized. 497 + /// 498 + /// # Safety 499 + /// 500 + /// The caller guarantees that the value behind this pointer has been initialized. It is 501 + /// *immediate* UB to call this when the value is not initialized. 502 + pub unsafe fn assume_init(self) -> UniqueArc<T> { 529 503 let inner = ManuallyDrop::new(self).inner.ptr; 530 504 UniqueArc { 531 505 // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be 532 506 // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`. 533 507 inner: unsafe { Arc::from_inner(inner.cast()) }, 508 + } 509 + } 510 + 511 + /// Initialize `self` using the given initializer. 512 + pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> { 513 + // SAFETY: The supplied pointer is valid for initialization. 514 + match unsafe { init.__init(self.as_mut_ptr()) } { 515 + // SAFETY: Initialization completed successfully. 516 + Ok(()) => Ok(unsafe { self.assume_init() }), 517 + Err(err) => Err(err), 518 + } 519 + } 520 + 521 + /// Pin-initialize `self` using the given pin-initializer. 522 + pub fn pin_init_with<E>( 523 + mut self, 524 + init: impl PinInit<T, E>, 525 + ) -> core::result::Result<Pin<UniqueArc<T>>, E> { 526 + // SAFETY: The supplied pointer is valid for initialization and we will later pin the value 527 + // to ensure it does not move. 528 + match unsafe { init.__pinned_init(self.as_mut_ptr()) } { 529 + // SAFETY: Initialization completed successfully. 530 + Ok(()) => Ok(unsafe { self.assume_init() }.into()), 531 + Err(err) => Err(err), 534 532 } 535 533 } 536 534 } ··· 592 520 // it is safe to dereference it. Additionally, we know there is only one reference when 593 521 // it's inside a `UniqueArc`, so it is safe to get a mutable reference. 594 522 unsafe { &mut self.inner.ptr.as_mut().data } 523 + } 524 + } 525 + 526 + impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> { 527 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 528 + fmt::Display::fmt(self.deref(), f) 529 + } 530 + } 531 + 532 + impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> { 533 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 534 + fmt::Display::fmt(self.deref(), f) 535 + } 536 + } 537 + 538 + impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> { 539 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 540 + fmt::Debug::fmt(self.deref(), f) 541 + } 542 + } 543 + 544 + impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> { 545 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 546 + fmt::Debug::fmt(self.deref(), f) 595 547 } 596 548 }

+28

rust/kernel/sync/arc/std_vendor.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + //! The contents of this file come from the Rust standard library, hosted in 4 + //! the <https://github.com/rust-lang/rust> repository, licensed under 5 + //! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, 6 + //! see <https://github.com/rust-lang/rust/blob/master/COPYRIGHT>. 7 + 8 + use crate::sync::{arc::ArcInner, Arc}; 9 + use core::any::Any; 10 + 11 + impl Arc<dyn Any + Send + Sync> { 12 + /// Attempt to downcast the `Arc<dyn Any + Send + Sync>` to a concrete type. 13 + pub fn downcast<T>(self) -> core::result::Result<Arc<T>, Self> 14 + where 15 + T: Any + Send + Sync, 16 + { 17 + if (*self).is::<T>() { 18 + // SAFETY: We have just checked that the type is correct, so we can cast the pointer. 19 + unsafe { 20 + let ptr = self.ptr.cast::<ArcInner<T>>(); 21 + core::mem::forget(self); 22 + Ok(Arc::from_inner(ptr)) 23 + } 24 + } else { 25 + Err(self) 26 + } 27 + } 28 + }

+174

rust/kernel/sync/condvar.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + //! A condition variable. 4 + //! 5 + //! This module allows Rust code to use the kernel's [`struct wait_queue_head`] as a condition 6 + //! variable. 7 + 8 + use super::{lock::Backend, lock::Guard, LockClassKey}; 9 + use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque}; 10 + use core::marker::PhantomPinned; 11 + use macros::pin_data; 12 + 13 + /// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class. 14 + #[macro_export] 15 + macro_rules! new_condvar { 16 + ($($name:literal)?) => { 17 + $crate::sync::CondVar::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) 18 + }; 19 + } 20 + 21 + /// A conditional variable. 22 + /// 23 + /// Exposes the kernel's [`struct wait_queue_head`] as a condition variable. It allows the caller to 24 + /// atomically release the given lock and go to sleep. It reacquires the lock when it wakes up. And 25 + /// it wakes up when notified by another thread (via [`CondVar::notify_one`] or 26 + /// [`CondVar::notify_all`]) or because the thread received a signal. It may also wake up 27 + /// spuriously. 28 + /// 29 + /// Instances of [`CondVar`] need a lock class and to be pinned. The recommended way to create such 30 + /// instances is with the [`pin_init`](crate::pin_init) and [`new_condvar`] macros. 31 + /// 32 + /// # Examples 33 + /// 34 + /// The following is an example of using a condvar with a mutex: 35 + /// 36 + /// ``` 37 + /// use kernel::sync::{CondVar, Mutex}; 38 + /// use kernel::{new_condvar, new_mutex}; 39 + /// 40 + /// #[pin_data] 41 + /// pub struct Example { 42 + /// #[pin] 43 + /// value: Mutex<u32>, 44 + /// 45 + /// #[pin] 46 + /// value_changed: CondVar, 47 + /// } 48 + /// 49 + /// /// Waits for `e.value` to become `v`. 50 + /// fn wait_for_value(e: &Example, v: u32) { 51 + /// let mut guard = e.value.lock(); 52 + /// while *guard != v { 53 + /// e.value_changed.wait_uninterruptible(&mut guard); 54 + /// } 55 + /// } 56 + /// 57 + /// /// Increments `e.value` and notifies all potential waiters. 58 + /// fn increment(e: &Example) { 59 + /// *e.value.lock() += 1; 60 + /// e.value_changed.notify_all(); 61 + /// } 62 + /// 63 + /// /// Allocates a new boxed `Example`. 64 + /// fn new_example() -> Result<Pin<Box<Example>>> { 65 + /// Box::pin_init(pin_init!(Example { 66 + /// value <- new_mutex!(0), 67 + /// value_changed <- new_condvar!(), 68 + /// })) 69 + /// } 70 + /// ``` 71 + /// 72 + /// [`struct wait_queue_head`]: ../../../include/linux/wait.h 73 + #[pin_data] 74 + pub struct CondVar { 75 + #[pin] 76 + pub(crate) wait_list: Opaque<bindings::wait_queue_head>, 77 + 78 + /// A condvar needs to be pinned because it contains a [`struct list_head`] that is 79 + /// self-referential, so it cannot be safely moved once it is initialised. 80 + #[pin] 81 + _pin: PhantomPinned, 82 + } 83 + 84 + // SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread. 85 + #[allow(clippy::non_send_fields_in_send_ty)] 86 + unsafe impl Send for CondVar {} 87 + 88 + // SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads 89 + // concurrently. 90 + unsafe impl Sync for CondVar {} 91 + 92 + impl CondVar { 93 + /// Constructs a new condvar initialiser. 94 + #[allow(clippy::new_ret_no_self)] 95 + pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { 96 + pin_init!(Self { 97 + _pin: PhantomPinned, 98 + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have 99 + // static lifetimes so they live indefinitely. 100 + wait_list <- Opaque::ffi_init(|slot| unsafe { 101 + bindings::__init_waitqueue_head(slot, name.as_char_ptr(), key.as_ptr()) 102 + }), 103 + }) 104 + } 105 + 106 + fn wait_internal<T: ?Sized, B: Backend>(&self, wait_state: u32, guard: &mut Guard<'_, T, B>) { 107 + let wait = Opaque::<bindings::wait_queue_entry>::uninit(); 108 + 109 + // SAFETY: `wait` points to valid memory. 110 + unsafe { bindings::init_wait(wait.get()) }; 111 + 112 + // SAFETY: Both `wait` and `wait_list` point to valid memory. 113 + unsafe { 114 + bindings::prepare_to_wait_exclusive(self.wait_list.get(), wait.get(), wait_state as _) 115 + }; 116 + 117 + // SAFETY: No arguments, switches to another thread. 118 + guard.do_unlocked(|| unsafe { bindings::schedule() }); 119 + 120 + // SAFETY: Both `wait` and `wait_list` point to valid memory. 121 + unsafe { bindings::finish_wait(self.wait_list.get(), wait.get()) }; 122 + } 123 + 124 + /// Releases the lock and waits for a notification in interruptible mode. 125 + /// 126 + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the 127 + /// thread to sleep, reacquiring the lock on wake up. It wakes up when notified by 128 + /// [`CondVar::notify_one`] or [`CondVar::notify_all`], or when the thread receives a signal. 129 + /// It may also wake up spuriously. 130 + /// 131 + /// Returns whether there is a signal pending. 132 + #[must_use = "wait returns if a signal is pending, so the caller must check the return value"] 133 + pub fn wait<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) -> bool { 134 + self.wait_internal(bindings::TASK_INTERRUPTIBLE, guard); 135 + crate::current!().signal_pending() 136 + } 137 + 138 + /// Releases the lock and waits for a notification in uninterruptible mode. 139 + /// 140 + /// Similar to [`CondVar::wait`], except that the wait is not interruptible. That is, the 141 + /// thread won't wake up due to signals. It may, however, wake up supirously. 142 + pub fn wait_uninterruptible<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) { 143 + self.wait_internal(bindings::TASK_UNINTERRUPTIBLE, guard) 144 + } 145 + 146 + /// Calls the kernel function to notify the appropriate number of threads with the given flags. 147 + fn notify(&self, count: i32, flags: u32) { 148 + // SAFETY: `wait_list` points to valid memory. 149 + unsafe { 150 + bindings::__wake_up( 151 + self.wait_list.get(), 152 + bindings::TASK_NORMAL, 153 + count, 154 + flags as _, 155 + ) 156 + }; 157 + } 158 + 159 + /// Wakes a single waiter up, if any. 160 + /// 161 + /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost 162 + /// completely (as opposed to automatically waking up the next waiter). 163 + pub fn notify_one(&self) { 164 + self.notify(1, 0); 165 + } 166 + 167 + /// Wakes all waiters up, if any. 168 + /// 169 + /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost 170 + /// completely (as opposed to automatically waking up the next waiter). 171 + pub fn notify_all(&self) { 172 + self.notify(0, 0); 173 + } 174 + }

+191

rust/kernel/sync/lock.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + //! Generic kernel lock and guard. 4 + //! 5 + //! It contains a generic Rust lock and guard that allow for different backends (e.g., mutexes, 6 + //! spinlocks, raw spinlocks) to be provided with minimal effort. 7 + 8 + use super::LockClassKey; 9 + use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard}; 10 + use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; 11 + use macros::pin_data; 12 + 13 + pub mod mutex; 14 + pub mod spinlock; 15 + 16 + /// The "backend" of a lock. 17 + /// 18 + /// It is the actual implementation of the lock, without the need to repeat patterns used in all 19 + /// locks. 20 + /// 21 + /// # Safety 22 + /// 23 + /// - Implementers must ensure that only one thread/CPU may access the protected data once the lock 24 + /// is owned, that is, between calls to `lock` and `unlock`. 25 + /// - Implementers must also ensure that `relock` uses the same locking method as the original 26 + /// lock operation. 27 + pub unsafe trait Backend { 28 + /// The state required by the lock. 29 + type State; 30 + 31 + /// The state required to be kept between lock and unlock. 32 + type GuardState; 33 + 34 + /// Initialises the lock. 35 + /// 36 + /// # Safety 37 + /// 38 + /// `ptr` must be valid for write for the duration of the call, while `name` and `key` must 39 + /// remain valid for read indefinitely. 40 + unsafe fn init( 41 + ptr: *mut Self::State, 42 + name: *const core::ffi::c_char, 43 + key: *mut bindings::lock_class_key, 44 + ); 45 + 46 + /// Acquires the lock, making the caller its owner. 47 + /// 48 + /// # Safety 49 + /// 50 + /// Callers must ensure that [`Backend::init`] has been previously called. 51 + #[must_use] 52 + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState; 53 + 54 + /// Releases the lock, giving up its ownership. 55 + /// 56 + /// # Safety 57 + /// 58 + /// It must only be called by the current owner of the lock. 59 + unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState); 60 + 61 + /// Reacquires the lock, making the caller its owner. 62 + /// 63 + /// # Safety 64 + /// 65 + /// Callers must ensure that `guard_state` comes from a previous call to [`Backend::lock`] (or 66 + /// variant) that has been unlocked with [`Backend::unlock`] and will be relocked now. 67 + unsafe fn relock(ptr: *mut Self::State, guard_state: &mut Self::GuardState) { 68 + // SAFETY: The safety requirements ensure that the lock is initialised. 69 + *guard_state = unsafe { Self::lock(ptr) }; 70 + } 71 + } 72 + 73 + /// A mutual exclusion primitive. 74 + /// 75 + /// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock backend 76 + /// specified as the generic parameter `B`. 77 + #[pin_data] 78 + pub struct Lock<T: ?Sized, B: Backend> { 79 + /// The kernel lock object. 80 + #[pin] 81 + state: Opaque<B::State>, 82 + 83 + /// Some locks are known to be self-referential (e.g., mutexes), while others are architecture 84 + /// or config defined (e.g., spinlocks). So we conservatively require them to be pinned in case 85 + /// some architecture uses self-references now or in the future. 86 + #[pin] 87 + _pin: PhantomPinned, 88 + 89 + /// The data protected by the lock. 90 + pub(crate) data: UnsafeCell<T>, 91 + } 92 + 93 + // SAFETY: `Lock` can be transferred across thread boundaries iff the data it protects can. 94 + unsafe impl<T: ?Sized + Send, B: Backend> Send for Lock<T, B> {} 95 + 96 + // SAFETY: `Lock` serialises the interior mutability it provides, so it is `Sync` as long as the 97 + // data it protects is `Send`. 98 + unsafe impl<T: ?Sized + Send, B: Backend> Sync for Lock<T, B> {} 99 + 100 + impl<T, B: Backend> Lock<T, B> { 101 + /// Constructs a new lock initialiser. 102 + #[allow(clippy::new_ret_no_self)] 103 + pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { 104 + pin_init!(Self { 105 + data: UnsafeCell::new(t), 106 + _pin: PhantomPinned, 107 + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have 108 + // static lifetimes so they live indefinitely. 109 + state <- Opaque::ffi_init(|slot| unsafe { 110 + B::init(slot, name.as_char_ptr(), key.as_ptr()) 111 + }), 112 + }) 113 + } 114 + } 115 + 116 + impl<T: ?Sized, B: Backend> Lock<T, B> { 117 + /// Acquires the lock and gives the caller access to the data protected by it. 118 + pub fn lock(&self) -> Guard<'_, T, B> { 119 + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves 120 + // that `init` was called. 121 + let state = unsafe { B::lock(self.state.get()) }; 122 + // SAFETY: The lock was just acquired. 123 + unsafe { Guard::new(self, state) } 124 + } 125 + } 126 + 127 + /// A lock guard. 128 + /// 129 + /// Allows mutual exclusion primitives that implement the `Backend` trait to automatically unlock 130 + /// when a guard goes out of scope. It also provides a safe and convenient way to access the data 131 + /// protected by the lock. 132 + #[must_use = "the lock unlocks immediately when the guard is unused"] 133 + pub struct Guard<'a, T: ?Sized, B: Backend> { 134 + pub(crate) lock: &'a Lock<T, B>, 135 + pub(crate) state: B::GuardState, 136 + _not_send: PhantomData<*mut ()>, 137 + } 138 + 139 + // SAFETY: `Guard` is sync when the data protected by the lock is also sync. 140 + unsafe impl<T: Sync + ?Sized, B: Backend> Sync for Guard<'_, T, B> {} 141 + 142 + impl<T: ?Sized, B: Backend> Guard<'_, T, B> { 143 + pub(crate) fn do_unlocked(&mut self, cb: impl FnOnce()) { 144 + // SAFETY: The caller owns the lock, so it is safe to unlock it. 145 + unsafe { B::unlock(self.lock.state.get(), &self.state) }; 146 + 147 + // SAFETY: The lock was just unlocked above and is being relocked now. 148 + let _relock = 149 + ScopeGuard::new(|| unsafe { B::relock(self.lock.state.get(), &mut self.state) }); 150 + 151 + cb(); 152 + } 153 + } 154 + 155 + impl<T: ?Sized, B: Backend> core::ops::Deref for Guard<'_, T, B> { 156 + type Target = T; 157 + 158 + fn deref(&self) -> &Self::Target { 159 + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. 160 + unsafe { &*self.lock.data.get() } 161 + } 162 + } 163 + 164 + impl<T: ?Sized, B: Backend> core::ops::DerefMut for Guard<'_, T, B> { 165 + fn deref_mut(&mut self) -> &mut Self::Target { 166 + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. 167 + unsafe { &mut *self.lock.data.get() } 168 + } 169 + } 170 + 171 + impl<T: ?Sized, B: Backend> Drop for Guard<'_, T, B> { 172 + fn drop(&mut self) { 173 + // SAFETY: The caller owns the lock, so it is safe to unlock it. 174 + unsafe { B::unlock(self.lock.state.get(), &self.state) }; 175 + } 176 + } 177 + 178 + impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { 179 + /// Constructs a new immutable lock guard. 180 + /// 181 + /// # Safety 182 + /// 183 + /// The caller must ensure that it owns the lock. 184 + pub(crate) unsafe fn new(lock: &'a Lock<T, B>, state: B::GuardState) -> Self { 185 + Self { 186 + lock, 187 + state, 188 + _not_send: PhantomData, 189 + } 190 + } 191 + }

+118

rust/kernel/sync/lock/mutex.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + //! A kernel mutex. 4 + //! 5 + //! This module allows Rust code to use the kernel's `struct mutex`. 6 + 7 + use crate::bindings; 8 + 9 + /// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class. 10 + /// 11 + /// It uses the name if one is given, otherwise it generates one based on the file name and line 12 + /// number. 13 + #[macro_export] 14 + macro_rules! new_mutex { 15 + ($inner:expr $(, $name:literal)? $(,)?) => { 16 + $crate::sync::Mutex::new( 17 + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) 18 + }; 19 + } 20 + 21 + /// A mutual exclusion primitive. 22 + /// 23 + /// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, 24 + /// only one at a time is allowed to progress, the others will block (sleep) until the mutex is 25 + /// unlocked, at which point another thread will be allowed to wake up and make progress. 26 + /// 27 + /// Since it may block, [`Mutex`] needs to be used with care in atomic contexts. 28 + /// 29 + /// Instances of [`Mutex`] need a lock class and to be pinned. The recommended way to create such 30 + /// instances is with the [`pin_init`](crate::pin_init) and [`new_mutex`] macros. 31 + /// 32 + /// # Examples 33 + /// 34 + /// The following example shows how to declare, allocate and initialise a struct (`Example`) that 35 + /// contains an inner struct (`Inner`) that is protected by a mutex. 36 + /// 37 + /// ``` 38 + /// use kernel::{init::InPlaceInit, init::PinInit, new_mutex, pin_init, sync::Mutex}; 39 + /// 40 + /// struct Inner { 41 + /// a: u32, 42 + /// b: u32, 43 + /// } 44 + /// 45 + /// #[pin_data] 46 + /// struct Example { 47 + /// c: u32, 48 + /// #[pin] 49 + /// d: Mutex<Inner>, 50 + /// } 51 + /// 52 + /// impl Example { 53 + /// fn new() -> impl PinInit<Self> { 54 + /// pin_init!(Self { 55 + /// c: 10, 56 + /// d <- new_mutex!(Inner { a: 20, b: 30 }), 57 + /// }) 58 + /// } 59 + /// } 60 + /// 61 + /// // Allocate a boxed `Example`. 62 + /// let e = Box::pin_init(Example::new())?; 63 + /// assert_eq!(e.c, 10); 64 + /// assert_eq!(e.d.lock().a, 20); 65 + /// assert_eq!(e.d.lock().b, 30); 66 + /// ``` 67 + /// 68 + /// The following example shows how to use interior mutability to modify the contents of a struct 69 + /// protected by a mutex despite only having a shared reference: 70 + /// 71 + /// ``` 72 + /// use kernel::sync::Mutex; 73 + /// 74 + /// struct Example { 75 + /// a: u32, 76 + /// b: u32, 77 + /// } 78 + /// 79 + /// fn example(m: &Mutex<Example>) { 80 + /// let mut guard = m.lock(); 81 + /// guard.a += 10; 82 + /// guard.b += 20; 83 + /// } 84 + /// ``` 85 + /// 86 + /// [`struct mutex`]: ../../../../include/linux/mutex.h 87 + pub type Mutex<T> = super::Lock<T, MutexBackend>; 88 + 89 + /// A kernel `struct mutex` lock backend. 90 + pub struct MutexBackend; 91 + 92 + // SAFETY: The underlying kernel `struct mutex` object ensures mutual exclusion. 93 + unsafe impl super::Backend for MutexBackend { 94 + type State = bindings::mutex; 95 + type GuardState = (); 96 + 97 + unsafe fn init( 98 + ptr: *mut Self::State, 99 + name: *const core::ffi::c_char, 100 + key: *mut bindings::lock_class_key, 101 + ) { 102 + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and 103 + // `key` are valid for read indefinitely. 104 + unsafe { bindings::__mutex_init(ptr, name, key) } 105 + } 106 + 107 + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { 108 + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid 109 + // memory, and that it has been initialised before. 110 + unsafe { bindings::mutex_lock(ptr) }; 111 + } 112 + 113 + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { 114 + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the 115 + // caller is the owner of the mutex. 116 + unsafe { bindings::mutex_unlock(ptr) }; 117 + } 118 + }

+117

rust/kernel/sync/lock/spinlock.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + //! A kernel spinlock. 4 + //! 5 + //! This module allows Rust code to use the kernel's `spinlock_t`. 6 + 7 + use crate::bindings; 8 + 9 + /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. 10 + /// 11 + /// It uses the name if one is given, otherwise it generates one based on the file name and line 12 + /// number. 13 + #[macro_export] 14 + macro_rules! new_spinlock { 15 + ($inner:expr $(, $name:literal)? $(,)?) => { 16 + $crate::sync::SpinLock::new( 17 + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) 18 + }; 19 + } 20 + 21 + /// A spinlock. 22 + /// 23 + /// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only 24 + /// one at a time is allowed to progress, the others will block (spinning) until the spinlock is 25 + /// unlocked, at which point another CPU will be allowed to make progress. 26 + /// 27 + /// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such 28 + /// instances is with the [`pin_init`](crate::pin_init) and [`new_spinlock`] macros. 29 + /// 30 + /// # Examples 31 + /// 32 + /// The following example shows how to declare, allocate and initialise a struct (`Example`) that 33 + /// contains an inner struct (`Inner`) that is protected by a spinlock. 34 + /// 35 + /// ``` 36 + /// use kernel::{init::InPlaceInit, init::PinInit, new_spinlock, pin_init, sync::SpinLock}; 37 + /// 38 + /// struct Inner { 39 + /// a: u32, 40 + /// b: u32, 41 + /// } 42 + /// 43 + /// #[pin_data] 44 + /// struct Example { 45 + /// c: u32, 46 + /// #[pin] 47 + /// d: SpinLock<Inner>, 48 + /// } 49 + /// 50 + /// impl Example { 51 + /// fn new() -> impl PinInit<Self> { 52 + /// pin_init!(Self { 53 + /// c: 10, 54 + /// d <- new_spinlock!(Inner { a: 20, b: 30 }), 55 + /// }) 56 + /// } 57 + /// } 58 + /// 59 + /// // Allocate a boxed `Example`. 60 + /// let e = Box::pin_init(Example::new())?; 61 + /// assert_eq!(e.c, 10); 62 + /// assert_eq!(e.d.lock().a, 20); 63 + /// assert_eq!(e.d.lock().b, 30); 64 + /// ``` 65 + /// 66 + /// The following example shows how to use interior mutability to modify the contents of a struct 67 + /// protected by a spinlock despite only having a shared reference: 68 + /// 69 + /// ``` 70 + /// use kernel::sync::SpinLock; 71 + /// 72 + /// struct Example { 73 + /// a: u32, 74 + /// b: u32, 75 + /// } 76 + /// 77 + /// fn example(m: &SpinLock<Example>) { 78 + /// let mut guard = m.lock(); 79 + /// guard.a += 10; 80 + /// guard.b += 20; 81 + /// } 82 + /// ``` 83 + /// 84 + /// [`spinlock_t`]: ../../../../include/linux/spinlock.h 85 + pub type SpinLock<T> = super::Lock<T, SpinLockBackend>; 86 + 87 + /// A kernel `spinlock_t` lock backend. 88 + pub struct SpinLockBackend; 89 + 90 + // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the 91 + // default implementation that always calls the same locking method. 92 + unsafe impl super::Backend for SpinLockBackend { 93 + type State = bindings::spinlock_t; 94 + type GuardState = (); 95 + 96 + unsafe fn init( 97 + ptr: *mut Self::State, 98 + name: *const core::ffi::c_char, 99 + key: *mut bindings::lock_class_key, 100 + ) { 101 + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and 102 + // `key` are valid for read indefinitely. 103 + unsafe { bindings::__spin_lock_init(ptr, name, key) } 104 + } 105 + 106 + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { 107 + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid 108 + // memory, and that it has been initialised before. 109 + unsafe { bindings::spin_lock(ptr) } 110 + } 111 + 112 + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { 113 + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the 114 + // caller is the owner of the mutex. 115 + unsafe { bindings::spin_unlock(ptr) } 116 + } 117 + }

+156

rust/kernel/sync/locked_by.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + //! A wrapper for data protected by a lock that does not wrap it. 4 + 5 + use super::{lock::Backend, lock::Lock}; 6 + use crate::build_assert; 7 + use core::{cell::UnsafeCell, mem::size_of, ptr}; 8 + 9 + /// Allows access to some data to be serialised by a lock that does not wrap it. 10 + /// 11 + /// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g., 12 + /// [`super::Mutex`] or [`super::SpinLock`]. [`LockedBy`] is meant for cases when this is not 13 + /// possible. For example, if a container has a lock and some data in the contained elements needs 14 + /// to be protected by the same lock. 15 + /// 16 + /// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it 17 + /// when the caller shows evidence that the 'external' lock is locked. It panics if the evidence 18 + /// refers to the wrong instance of the lock. 19 + /// 20 + /// # Examples 21 + /// 22 + /// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an 23 + /// aggregate of all `InnerFile::bytes_used` and must be kept consistent; so we wrap `InnerFile` in 24 + /// a `LockedBy` so that it shares a lock with `InnerDirectory`. This allows us to enforce at 25 + /// compile-time that access to `InnerFile` is only granted when an `InnerDirectory` is also 26 + /// locked; we enforce at run time that the right `InnerDirectory` is locked. 27 + /// 28 + /// ``` 29 + /// use kernel::sync::{LockedBy, Mutex}; 30 + /// 31 + /// struct InnerFile { 32 + /// bytes_used: u64, 33 + /// } 34 + /// 35 + /// struct File { 36 + /// _ino: u32, 37 + /// inner: LockedBy<InnerFile, InnerDirectory>, 38 + /// } 39 + /// 40 + /// struct InnerDirectory { 41 + /// /// The sum of the bytes used by all files. 42 + /// bytes_used: u64, 43 + /// _files: Vec<File>, 44 + /// } 45 + /// 46 + /// struct Directory { 47 + /// _ino: u32, 48 + /// inner: Mutex<InnerDirectory>, 49 + /// } 50 + /// 51 + /// /// Prints `bytes_used` from both the directory and file. 52 + /// fn print_bytes_used(dir: &Directory, file: &File) { 53 + /// let guard = dir.inner.lock(); 54 + /// let inner_file = file.inner.access(&guard); 55 + /// pr_info!("{} {}", guard.bytes_used, inner_file.bytes_used); 56 + /// } 57 + /// 58 + /// /// Increments `bytes_used` for both the directory and file. 59 + /// fn inc_bytes_used(dir: &Directory, file: &File) { 60 + /// let mut guard = dir.inner.lock(); 61 + /// guard.bytes_used += 10; 62 + /// 63 + /// let file_inner = file.inner.access_mut(&mut guard); 64 + /// file_inner.bytes_used += 10; 65 + /// } 66 + /// 67 + /// /// Creates a new file. 68 + /// fn new_file(ino: u32, dir: &Directory) -> File { 69 + /// File { 70 + /// _ino: ino, 71 + /// inner: LockedBy::new(&dir.inner, InnerFile { bytes_used: 0 }), 72 + /// } 73 + /// } 74 + /// ``` 75 + pub struct LockedBy<T: ?Sized, U: ?Sized> { 76 + owner: *const U, 77 + data: UnsafeCell<T>, 78 + } 79 + 80 + // SAFETY: `LockedBy` can be transferred across thread boundaries iff the data it protects can. 81 + unsafe impl<T: ?Sized + Send, U: ?Sized> Send for LockedBy<T, U> {} 82 + 83 + // SAFETY: `LockedBy` serialises the interior mutability it provides, so it is `Sync` as long as the 84 + // data it protects is `Send`. 85 + unsafe impl<T: ?Sized + Send, U: ?Sized> Sync for LockedBy<T, U> {} 86 + 87 + impl<T, U> LockedBy<T, U> { 88 + /// Constructs a new instance of [`LockedBy`]. 89 + /// 90 + /// It stores a raw pointer to the owner that is never dereferenced. It is only used to ensure 91 + /// that the right owner is being used to access the protected data. If the owner is freed, the 92 + /// data becomes inaccessible; if another instance of the owner is allocated *on the same 93 + /// memory location*, the data becomes accessible again: none of this affects memory safety 94 + /// because in any case at most one thread (or CPU) can access the protected data at a time. 95 + pub fn new<B: Backend>(owner: &Lock<U, B>, data: T) -> Self { 96 + build_assert!( 97 + size_of::<Lock<U, B>>() > 0, 98 + "The lock type cannot be a ZST because it may be impossible to distinguish instances" 99 + ); 100 + Self { 101 + owner: owner.data.get(), 102 + data: UnsafeCell::new(data), 103 + } 104 + } 105 + } 106 + 107 + impl<T: ?Sized, U> LockedBy<T, U> { 108 + /// Returns a reference to the protected data when the caller provides evidence (via a 109 + /// reference) that the owner is locked. 110 + /// 111 + /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&U` that matches 112 + /// the data protected by the lock without actually holding it. 113 + /// 114 + /// # Panics 115 + /// 116 + /// Panics if `owner` is different from the data protected by the lock used in 117 + /// [`new`](LockedBy::new). 118 + pub fn access<'a>(&'a self, owner: &'a U) -> &'a T { 119 + build_assert!( 120 + size_of::<U>() > 0, 121 + "`U` cannot be a ZST because `owner` wouldn't be unique" 122 + ); 123 + if !ptr::eq(owner, self.owner) { 124 + panic!("mismatched owners"); 125 + } 126 + 127 + // SAFETY: `owner` is evidence that the owner is locked. 128 + unsafe { &*self.data.get() } 129 + } 130 + 131 + /// Returns a mutable reference to the protected data when the caller provides evidence (via a 132 + /// mutable owner) that the owner is locked mutably. 133 + /// 134 + /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&mut U` that 135 + /// matches the data protected by the lock without actually holding it. 136 + /// 137 + /// Showing a mutable reference to the owner is sufficient because we know no other references 138 + /// can exist to it. 139 + /// 140 + /// # Panics 141 + /// 142 + /// Panics if `owner` is different from the data protected by the lock used in 143 + /// [`new`](LockedBy::new). 144 + pub fn access_mut<'a>(&'a self, owner: &'a mut U) -> &'a mut T { 145 + build_assert!( 146 + size_of::<U>() > 0, 147 + "`U` cannot be a ZST because `owner` wouldn't be unique" 148 + ); 149 + if !ptr::eq(owner, self.owner) { 150 + panic!("mismatched owners"); 151 + } 152 + 153 + // SAFETY: `owner` is evidence that there is only one reference to the owner. 154 + unsafe { &mut *self.data.get() } 155 + } 156 + }

+155

rust/kernel/task.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + //! Tasks (threads and processes). 4 + //! 5 + //! C header: [`include/linux/sched.h`](../../../../include/linux/sched.h). 6 + 7 + use crate::{bindings, types::Opaque}; 8 + use core::{marker::PhantomData, ops::Deref, ptr}; 9 + 10 + /// Returns the currently running task. 11 + #[macro_export] 12 + macro_rules! current { 13 + () => { 14 + // SAFETY: Deref + addr-of below create a temporary `TaskRef` that cannot outlive the 15 + // caller. 16 + unsafe { &*$crate::task::Task::current() } 17 + }; 18 + } 19 + 20 + /// Wraps the kernel's `struct task_struct`. 21 + /// 22 + /// # Invariants 23 + /// 24 + /// All instances are valid tasks created by the C portion of the kernel. 25 + /// 26 + /// Instances of this type are always ref-counted, that is, a call to `get_task_struct` ensures 27 + /// that the allocation remains valid at least until the matching call to `put_task_struct`. 28 + /// 29 + /// # Examples 30 + /// 31 + /// The following is an example of getting the PID of the current thread with zero additional cost 32 + /// when compared to the C version: 33 + /// 34 + /// ``` 35 + /// let pid = current!().pid(); 36 + /// ``` 37 + /// 38 + /// Getting the PID of the current process, also zero additional cost: 39 + /// 40 + /// ``` 41 + /// let pid = current!().group_leader().pid(); 42 + /// ``` 43 + /// 44 + /// Getting the current task and storing it in some struct. The reference count is automatically 45 + /// incremented when creating `State` and decremented when it is dropped: 46 + /// 47 + /// ``` 48 + /// use kernel::{task::Task, types::ARef}; 49 + /// 50 + /// struct State { 51 + /// creator: ARef<Task>, 52 + /// index: u32, 53 + /// } 54 + /// 55 + /// impl State { 56 + /// fn new() -> Self { 57 + /// Self { 58 + /// creator: current!().into(), 59 + /// index: 0, 60 + /// } 61 + /// } 62 + /// } 63 + /// ``` 64 + #[repr(transparent)] 65 + pub struct Task(pub(crate) Opaque<bindings::task_struct>); 66 + 67 + // SAFETY: It's OK to access `Task` through references from other threads because we're either 68 + // accessing properties that don't change (e.g., `pid`, `group_leader`) or that are properly 69 + // synchronised by C code (e.g., `signal_pending`). 70 + unsafe impl Sync for Task {} 71 + 72 + /// The type of process identifiers (PIDs). 73 + type Pid = bindings::pid_t; 74 + 75 + impl Task { 76 + /// Returns a task reference for the currently executing task/thread. 77 + /// 78 + /// The recommended way to get the current task/thread is to use the 79 + /// [`current`](crate::current) macro because it is safe. 80 + /// 81 + /// # Safety 82 + /// 83 + /// Callers must ensure that the returned object doesn't outlive the current task/thread. 84 + pub unsafe fn current() -> impl Deref<Target = Task> { 85 + struct TaskRef<'a> { 86 + task: &'a Task, 87 + _not_send: PhantomData<*mut ()>, 88 + } 89 + 90 + impl Deref for TaskRef<'_> { 91 + type Target = Task; 92 + 93 + fn deref(&self) -> &Self::Target { 94 + self.task 95 + } 96 + } 97 + 98 + // SAFETY: Just an FFI call with no additional safety requirements. 99 + let ptr = unsafe { bindings::get_current() }; 100 + 101 + TaskRef { 102 + // SAFETY: If the current thread is still running, the current task is valid. Given 103 + // that `TaskRef` is not `Send`, we know it cannot be transferred to another thread 104 + // (where it could potentially outlive the caller). 105 + task: unsafe { &*ptr.cast() }, 106 + _not_send: PhantomData, 107 + } 108 + } 109 + 110 + /// Returns the group leader of the given task. 111 + pub fn group_leader(&self) -> &Task { 112 + // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always 113 + // have a valid group_leader. 114 + let ptr = unsafe { *ptr::addr_of!((*self.0.get()).group_leader) }; 115 + 116 + // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`, 117 + // and given that a task has a reference to its group leader, we know it must be valid for 118 + // the lifetime of the returned task reference. 119 + unsafe { &*ptr.cast() } 120 + } 121 + 122 + /// Returns the PID of the given task. 123 + pub fn pid(&self) -> Pid { 124 + // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always 125 + // have a valid pid. 126 + unsafe { *ptr::addr_of!((*self.0.get()).pid) } 127 + } 128 + 129 + /// Determines whether the given task has pending signals. 130 + pub fn signal_pending(&self) -> bool { 131 + // SAFETY: By the type invariant, we know that `self.0` is valid. 132 + unsafe { bindings::signal_pending(self.0.get()) != 0 } 133 + } 134 + 135 + /// Wakes up the task. 136 + pub fn wake_up(&self) { 137 + // SAFETY: By the type invariant, we know that `self.0.get()` is non-null and valid. 138 + // And `wake_up_process` is safe to be called for any valid task, even if the task is 139 + // running. 140 + unsafe { bindings::wake_up_process(self.0.get()) }; 141 + } 142 + } 143 + 144 + // SAFETY: The type invariants guarantee that `Task` is always ref-counted. 145 + unsafe impl crate::types::AlwaysRefCounted for Task { 146 + fn inc_ref(&self) { 147 + // SAFETY: The existence of a shared reference means that the refcount is nonzero. 148 + unsafe { bindings::get_task_struct(self.0.get()) }; 149 + } 150 + 151 + unsafe fn dec_ref(obj: ptr::NonNull<Self>) { 152 + // SAFETY: The safety requirements guarantee that the refcount is nonzero. 153 + unsafe { bindings::put_task_struct(obj.cast().as_ptr()) } 154 + } 155 + }

+135

rust/kernel/types.rs

··· 2 2 3 3 //! Kernel types. 4 4 5 + use crate::init::{self, PinInit}; 5 6 use alloc::boxed::Box; 6 7 use core::{ 7 8 cell::UnsafeCell, 9 + marker::PhantomData, 8 10 mem::MaybeUninit, 9 11 ops::{Deref, DerefMut}, 12 + ptr::NonNull, 10 13 }; 11 14 12 15 /// Used to transfer ownership to and from foreign (non-Rust) languages. ··· 237 234 Self(MaybeUninit::uninit()) 238 235 } 239 236 237 + /// Creates a pin-initializer from the given initializer closure. 238 + /// 239 + /// The returned initializer calls the given closure with the pointer to the inner `T` of this 240 + /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it. 241 + /// 242 + /// This function is safe, because the `T` inside of an `Opaque` is allowed to be 243 + /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs 244 + /// to verify at that point that the inner value is valid. 245 + pub fn ffi_init(init_func: impl FnOnce(*mut T)) -> impl PinInit<Self> { 246 + // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully 247 + // initialize the `T`. 248 + unsafe { 249 + init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { 250 + init_func(Self::raw_get(slot)); 251 + Ok(()) 252 + }) 253 + } 254 + } 255 + 240 256 /// Returns a raw pointer to the opaque data. 241 257 pub fn get(&self) -> *mut T { 242 258 UnsafeCell::raw_get(self.0.as_ptr()) 259 + } 260 + 261 + /// Gets the value behind `this`. 262 + /// 263 + /// This function is useful to get access to the value without creating intermediate 264 + /// references. 265 + pub const fn raw_get(this: *const Self) -> *mut T { 266 + UnsafeCell::raw_get(this.cast::<UnsafeCell<T>>()) 267 + } 268 + } 269 + 270 + /// Types that are _always_ reference counted. 271 + /// 272 + /// It allows such types to define their own custom ref increment and decrement functions. 273 + /// Additionally, it allows users to convert from a shared reference `&T` to an owned reference 274 + /// [`ARef<T>`]. 275 + /// 276 + /// This is usually implemented by wrappers to existing structures on the C side of the code. For 277 + /// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted 278 + /// instances of a type. 279 + /// 280 + /// # Safety 281 + /// 282 + /// Implementers must ensure that increments to the reference count keep the object alive in memory 283 + /// at least until matching decrements are performed. 284 + /// 285 + /// Implementers must also ensure that all instances are reference-counted. (Otherwise they 286 + /// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object 287 + /// alive.) 288 + pub unsafe trait AlwaysRefCounted { 289 + /// Increments the reference count on the object. 290 + fn inc_ref(&self); 291 + 292 + /// Decrements the reference count on the object. 293 + /// 294 + /// Frees the object when the count reaches zero. 295 + /// 296 + /// # Safety 297 + /// 298 + /// Callers must ensure that there was a previous matching increment to the reference count, 299 + /// and that the object is no longer used after its reference count is decremented (as it may 300 + /// result in the object being freed), unless the caller owns another increment on the refcount 301 + /// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls 302 + /// [`AlwaysRefCounted::dec_ref`] once). 303 + unsafe fn dec_ref(obj: NonNull<Self>); 304 + } 305 + 306 + /// An owned reference to an always-reference-counted object. 307 + /// 308 + /// The object's reference count is automatically decremented when an instance of [`ARef`] is 309 + /// dropped. It is also automatically incremented when a new instance is created via 310 + /// [`ARef::clone`]. 311 + /// 312 + /// # Invariants 313 + /// 314 + /// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In 315 + /// particular, the [`ARef`] instance owns an increment on the underlying object's reference count. 316 + pub struct ARef<T: AlwaysRefCounted> { 317 + ptr: NonNull<T>, 318 + _p: PhantomData<T>, 319 + } 320 + 321 + impl<T: AlwaysRefCounted> ARef<T> { 322 + /// Creates a new instance of [`ARef`]. 323 + /// 324 + /// It takes over an increment of the reference count on the underlying object. 325 + /// 326 + /// # Safety 327 + /// 328 + /// Callers must ensure that the reference count was incremented at least once, and that they 329 + /// are properly relinquishing one increment. That is, if there is only one increment, callers 330 + /// must not use the underlying object anymore -- it is only safe to do so via the newly 331 + /// created [`ARef`]. 332 + pub unsafe fn from_raw(ptr: NonNull<T>) -> Self { 333 + // INVARIANT: The safety requirements guarantee that the new instance now owns the 334 + // increment on the refcount. 335 + Self { 336 + ptr, 337 + _p: PhantomData, 338 + } 339 + } 340 + } 341 + 342 + impl<T: AlwaysRefCounted> Clone for ARef<T> { 343 + fn clone(&self) -> Self { 344 + self.inc_ref(); 345 + // SAFETY: We just incremented the refcount above. 346 + unsafe { Self::from_raw(self.ptr) } 347 + } 348 + } 349 + 350 + impl<T: AlwaysRefCounted> Deref for ARef<T> { 351 + type Target = T; 352 + 353 + fn deref(&self) -> &Self::Target { 354 + // SAFETY: The type invariants guarantee that the object is valid. 355 + unsafe { self.ptr.as_ref() } 356 + } 357 + } 358 + 359 + impl<T: AlwaysRefCounted> From<&T> for ARef<T> { 360 + fn from(b: &T) -> Self { 361 + b.inc_ref(); 362 + // SAFETY: We just incremented the refcount above. 363 + unsafe { Self::from_raw(NonNull::from(b)) } 364 + } 365 + } 366 + 367 + impl<T: AlwaysRefCounted> Drop for ARef<T> { 368 + fn drop(&mut self) { 369 + // SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to 370 + // decrement. 371 + unsafe { T::dec_ref(self.ptr) }; 243 372 } 244 373 } 245 374

+9 -1

rust/macros/helpers.rs

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 3 - use proc_macro::{token_stream, TokenTree}; 3 + use proc_macro::{token_stream, Group, TokenTree}; 4 4 5 5 pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option<String> { 6 6 if let Some(TokenTree::Ident(ident)) = it.next() { ··· 54 54 let string = try_string(it).expect("Expected string"); 55 55 assert!(string.is_ascii(), "Expected ASCII string"); 56 56 string 57 + } 58 + 59 + pub(crate) fn expect_group(it: &mut token_stream::IntoIter) -> Group { 60 + if let TokenTree::Group(group) = it.next().expect("Reached end of token stream for Group") { 61 + group 62 + } else { 63 + panic!("Expected Group"); 64 + } 57 65 } 58 66 59 67 pub(crate) fn expect_end(it: &mut token_stream::IntoIter) {

+80

rust/macros/lib.rs

··· 2 2 3 3 //! Crate for all kernel procedural macros. 4 4 5 + #[macro_use] 6 + mod quote; 5 7 mod concat_idents; 6 8 mod helpers; 7 9 mod module; 10 + mod pin_data; 11 + mod pinned_drop; 8 12 mod vtable; 9 13 10 14 use proc_macro::TokenStream; ··· 169 165 #[proc_macro] 170 166 pub fn concat_idents(ts: TokenStream) -> TokenStream { 171 167 concat_idents::concat_idents(ts) 168 + } 169 + 170 + /// Used to specify the pinning information of the fields of a struct. 171 + /// 172 + /// This is somewhat similar in purpose as 173 + /// [pin-project-lite](https://crates.io/crates/pin-project-lite). 174 + /// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each 175 + /// field you want to structurally pin. 176 + /// 177 + /// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, 178 + /// then `#[pin]` directs the type of initializer that is required. 179 + /// 180 + /// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this 181 + /// macro, and change your `Drop` implementation to `PinnedDrop` annotated with 182 + /// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care. 183 + /// 184 + /// # Examples 185 + /// 186 + /// ```rust,ignore 187 + /// #[pin_data] 188 + /// struct DriverData { 189 + /// #[pin] 190 + /// queue: Mutex<Vec<Command>>, 191 + /// buf: Box<[u8; 1024 * 1024]>, 192 + /// } 193 + /// ``` 194 + /// 195 + /// ```rust,ignore 196 + /// #[pin_data(PinnedDrop)] 197 + /// struct DriverData { 198 + /// #[pin] 199 + /// queue: Mutex<Vec<Command>>, 200 + /// buf: Box<[u8; 1024 * 1024]>, 201 + /// raw_info: *mut Info, 202 + /// } 203 + /// 204 + /// #[pinned_drop] 205 + /// impl PinnedDrop for DriverData { 206 + /// fn drop(self: Pin<&mut Self>) { 207 + /// unsafe { bindings::destroy_info(self.raw_info) }; 208 + /// } 209 + /// } 210 + /// ``` 211 + /// 212 + /// [`pin_init!`]: ../kernel/macro.pin_init.html 213 + // ^ cannot use direct link, since `kernel` is not a dependency of `macros`. 214 + #[proc_macro_attribute] 215 + pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { 216 + pin_data::pin_data(inner, item) 217 + } 218 + 219 + /// Used to implement `PinnedDrop` safely. 220 + /// 221 + /// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`. 222 + /// 223 + /// # Examples 224 + /// 225 + /// ```rust,ignore 226 + /// #[pin_data(PinnedDrop)] 227 + /// struct DriverData { 228 + /// #[pin] 229 + /// queue: Mutex<Vec<Command>>, 230 + /// buf: Box<[u8; 1024 * 1024]>, 231 + /// raw_info: *mut Info, 232 + /// } 233 + /// 234 + /// #[pinned_drop] 235 + /// impl PinnedDrop for DriverData { 236 + /// fn drop(self: Pin<&mut Self>) { 237 + /// unsafe { bindings::destroy_info(self.raw_info) }; 238 + /// } 239 + /// } 240 + /// ``` 241 + #[proc_macro_attribute] 242 + pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { 243 + pinned_drop::pinned_drop(args, input) 172 244 }

+26 -6

rust/macros/module.rs

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 3 3 use crate::helpers::*; 4 - use proc_macro::{token_stream, Literal, TokenStream, TokenTree}; 4 + use proc_macro::{token_stream, Delimiter, Literal, TokenStream, TokenTree}; 5 5 use std::fmt::Write; 6 + 7 + fn expect_string_array(it: &mut token_stream::IntoIter) -> Vec<String> { 8 + let group = expect_group(it); 9 + assert_eq!(group.delimiter(), Delimiter::Bracket); 10 + let mut values = Vec::new(); 11 + let mut it = group.stream().into_iter(); 12 + 13 + while let Some(val) = try_string(&mut it) { 14 + assert!(val.is_ascii(), "Expected ASCII string"); 15 + values.push(val); 16 + match it.next() { 17 + Some(TokenTree::Punct(punct)) => assert_eq!(punct.as_char(), ','), 18 + None => break, 19 + _ => panic!("Expected ',' or end of array"), 20 + } 21 + } 22 + values 23 + } 6 24 7 25 struct ModInfoBuilder<'a> { 8 26 module: &'a str, ··· 96 78 name: String, 97 79 author: Option<String>, 98 80 description: Option<String>, 99 - alias: Option<String>, 81 + alias: Option<Vec<String>>, 100 82 } 101 83 102 84 impl ModuleInfo { ··· 130 112 "author" => info.author = Some(expect_string(it)), 131 113 "description" => info.description = Some(expect_string(it)), 132 114 "license" => info.license = expect_string_ascii(it), 133 - "alias" => info.alias = Some(expect_string_ascii(it)), 115 + "alias" => info.alias = Some(expect_string_array(it)), 134 116 _ => panic!( 135 117 "Unknown key \"{}\". Valid keys are: {:?}.", 136 118 key, EXPECTED_KEYS ··· 181 163 modinfo.emit("description", &description); 182 164 } 183 165 modinfo.emit("license", &info.license); 184 - if let Some(alias) = info.alias { 185 - modinfo.emit("alias", &alias); 166 + if let Some(aliases) = info.alias { 167 + for alias in aliases { 168 + modinfo.emit("alias", &alias); 169 + } 186 170 } 187 171 188 172 // Built-in modules also export the `file` modinfo string. ··· 278 258 return 0; 279 259 }} 280 260 Err(e) => {{ 281 - return e.to_kernel_errno(); 261 + return e.to_errno(); 282 262 }} 283 263 }} 284 264 }}

+79

rust/macros/pin_data.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + use proc_macro::{Punct, Spacing, TokenStream, TokenTree}; 4 + 5 + pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { 6 + // This proc-macro only does some pre-parsing and then delegates the actual parsing to 7 + // `kernel::__pin_data!`. 8 + // 9 + // In here we only collect the generics, since parsing them in declarative macros is very 10 + // elaborate. We also do not need to analyse their structure, we only need to collect them. 11 + 12 + // `impl_generics`, the declared generics with their bounds. 13 + let mut impl_generics = vec![]; 14 + // Only the names of the generics, without any bounds. 15 + let mut ty_generics = vec![]; 16 + // Tokens not related to the generics e.g. the `impl` token. 17 + let mut rest = vec![]; 18 + // The current level of `<`. 19 + let mut nesting = 0; 20 + let mut toks = input.into_iter(); 21 + // If we are at the beginning of a generic parameter. 22 + let mut at_start = true; 23 + for tt in &mut toks { 24 + match tt.clone() { 25 + TokenTree::Punct(p) if p.as_char() == '<' => { 26 + if nesting >= 1 { 27 + impl_generics.push(tt); 28 + } 29 + nesting += 1; 30 + } 31 + TokenTree::Punct(p) if p.as_char() == '>' => { 32 + if nesting == 0 { 33 + break; 34 + } else { 35 + nesting -= 1; 36 + if nesting >= 1 { 37 + impl_generics.push(tt); 38 + } 39 + if nesting == 0 { 40 + break; 41 + } 42 + } 43 + } 44 + tt => { 45 + if nesting == 1 { 46 + match &tt { 47 + TokenTree::Ident(i) if i.to_string() == "const" => {} 48 + TokenTree::Ident(_) if at_start => { 49 + ty_generics.push(tt.clone()); 50 + ty_generics.push(TokenTree::Punct(Punct::new(',', Spacing::Alone))); 51 + at_start = false; 52 + } 53 + TokenTree::Punct(p) if p.as_char() == ',' => at_start = true, 54 + TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { 55 + ty_generics.push(tt.clone()); 56 + } 57 + _ => {} 58 + } 59 + } 60 + if nesting >= 1 { 61 + impl_generics.push(tt); 62 + } else if nesting == 0 { 63 + rest.push(tt); 64 + } 65 + } 66 + } 67 + } 68 + rest.extend(toks); 69 + // This should be the body of the struct `{...}`. 70 + let last = rest.pop(); 71 + quote!(::kernel::__pin_data! { 72 + parse_input: 73 + @args(#args), 74 + @sig(#(#rest)*), 75 + @impl_generics(#(#impl_generics)*), 76 + @ty_generics(#(#ty_generics)*), 77 + @body(#last), 78 + }) 79 + }

+49

rust/macros/pinned_drop.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + use proc_macro::{TokenStream, TokenTree}; 4 + 5 + pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream { 6 + let mut toks = input.into_iter().collect::<Vec<_>>(); 7 + assert!(!toks.is_empty()); 8 + // Ensure that we have an `impl` item. 9 + assert!(matches!(&toks[0], TokenTree::Ident(i) if i.to_string() == "impl")); 10 + // Ensure that we are implementing `PinnedDrop`. 11 + let mut nesting: usize = 0; 12 + let mut pinned_drop_idx = None; 13 + for (i, tt) in toks.iter().enumerate() { 14 + match tt { 15 + TokenTree::Punct(p) if p.as_char() == '<' => { 16 + nesting += 1; 17 + } 18 + TokenTree::Punct(p) if p.as_char() == '>' => { 19 + nesting = nesting.checked_sub(1).unwrap(); 20 + continue; 21 + } 22 + _ => {} 23 + } 24 + if i >= 1 && nesting == 0 { 25 + // Found the end of the generics, this should be `PinnedDrop`. 26 + assert!( 27 + matches!(tt, TokenTree::Ident(i) if i.to_string() == "PinnedDrop"), 28 + "expected 'PinnedDrop', found: '{:?}'", 29 + tt 30 + ); 31 + pinned_drop_idx = Some(i); 32 + break; 33 + } 34 + } 35 + let idx = pinned_drop_idx 36 + .unwrap_or_else(|| panic!("Expected an `impl` block implementing `PinnedDrop`.")); 37 + // Fully qualify the `PinnedDrop`, as to avoid any tampering. 38 + toks.splice(idx..idx, quote!(::kernel::init::)); 39 + // Take the `{}` body and call the declarative macro. 40 + if let Some(TokenTree::Group(last)) = toks.pop() { 41 + let last = last.stream(); 42 + quote!(::kernel::__pinned_drop! { 43 + @impl_sig(#(#toks)*), 44 + @impl_body(#last), 45 + }) 46 + } else { 47 + TokenStream::from_iter(toks) 48 + } 49 + }

+143

rust/macros/quote.rs

··· 1 + // SPDX-License-Identifier: Apache-2.0 OR MIT 2 + 3 + use proc_macro::{TokenStream, TokenTree}; 4 + 5 + pub(crate) trait ToTokens { 6 + fn to_tokens(&self, tokens: &mut TokenStream); 7 + } 8 + 9 + impl<T: ToTokens> ToTokens for Option<T> { 10 + fn to_tokens(&self, tokens: &mut TokenStream) { 11 + if let Some(v) = self { 12 + v.to_tokens(tokens); 13 + } 14 + } 15 + } 16 + 17 + impl ToTokens for proc_macro::Group { 18 + fn to_tokens(&self, tokens: &mut TokenStream) { 19 + tokens.extend([TokenTree::from(self.clone())]); 20 + } 21 + } 22 + 23 + impl ToTokens for TokenTree { 24 + fn to_tokens(&self, tokens: &mut TokenStream) { 25 + tokens.extend([self.clone()]); 26 + } 27 + } 28 + 29 + impl ToTokens for TokenStream { 30 + fn to_tokens(&self, tokens: &mut TokenStream) { 31 + tokens.extend(self.clone()); 32 + } 33 + } 34 + 35 + /// Converts tokens into [`proc_macro::TokenStream`] and performs variable interpolations with 36 + /// the given span. 37 + /// 38 + /// This is a similar to the 39 + /// [`quote_spanned!`](https://docs.rs/quote/latest/quote/macro.quote_spanned.html) macro from the 40 + /// `quote` crate but provides only just enough functionality needed by the current `macros` crate. 41 + macro_rules! quote_spanned { 42 + ($span:expr => $($tt:tt)*) => { 43 + #[allow(clippy::vec_init_then_push)] 44 + { 45 + let mut tokens = ::std::vec::Vec::new(); 46 + let span = $span; 47 + quote_spanned!(@proc tokens span $($tt)*); 48 + ::proc_macro::TokenStream::from_iter(tokens) 49 + }}; 50 + (@proc $v:ident $span:ident) => {}; 51 + (@proc $v:ident $span:ident #$id:ident $($tt:tt)*) => { 52 + let mut ts = ::proc_macro::TokenStream::new(); 53 + $crate::quote::ToTokens::to_tokens(&$id, &mut ts); 54 + $v.extend(ts); 55 + quote_spanned!(@proc $v $span $($tt)*); 56 + }; 57 + (@proc $v:ident $span:ident #(#$id:ident)* $($tt:tt)*) => { 58 + for token in $id { 59 + let mut ts = ::proc_macro::TokenStream::new(); 60 + $crate::quote::ToTokens::to_tokens(&token, &mut ts); 61 + $v.extend(ts); 62 + } 63 + quote_spanned!(@proc $v $span $($tt)*); 64 + }; 65 + (@proc $v:ident $span:ident ( $($inner:tt)* ) $($tt:tt)*) => { 66 + let mut tokens = ::std::vec::Vec::new(); 67 + quote_spanned!(@proc tokens $span $($inner)*); 68 + $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new( 69 + ::proc_macro::Delimiter::Parenthesis, 70 + ::proc_macro::TokenStream::from_iter(tokens) 71 + ))); 72 + quote_spanned!(@proc $v $span $($tt)*); 73 + }; 74 + (@proc $v:ident $span:ident [ $($inner:tt)* ] $($tt:tt)*) => { 75 + let mut tokens = ::std::vec::Vec::new(); 76 + quote_spanned!(@proc tokens $span $($inner)*); 77 + $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new( 78 + ::proc_macro::Delimiter::Bracket, 79 + ::proc_macro::TokenStream::from_iter(tokens) 80 + ))); 81 + quote_spanned!(@proc $v $span $($tt)*); 82 + }; 83 + (@proc $v:ident $span:ident { $($inner:tt)* } $($tt:tt)*) => { 84 + let mut tokens = ::std::vec::Vec::new(); 85 + quote_spanned!(@proc tokens $span $($inner)*); 86 + $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new( 87 + ::proc_macro::Delimiter::Brace, 88 + ::proc_macro::TokenStream::from_iter(tokens) 89 + ))); 90 + quote_spanned!(@proc $v $span $($tt)*); 91 + }; 92 + (@proc $v:ident $span:ident :: $($tt:tt)*) => { 93 + $v.push(::proc_macro::TokenTree::Punct( 94 + ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Joint) 95 + )); 96 + $v.push(::proc_macro::TokenTree::Punct( 97 + ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Alone) 98 + )); 99 + quote_spanned!(@proc $v $span $($tt)*); 100 + }; 101 + (@proc $v:ident $span:ident : $($tt:tt)*) => { 102 + $v.push(::proc_macro::TokenTree::Punct( 103 + ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Alone) 104 + )); 105 + quote_spanned!(@proc $v $span $($tt)*); 106 + }; 107 + (@proc $v:ident $span:ident , $($tt:tt)*) => { 108 + $v.push(::proc_macro::TokenTree::Punct( 109 + ::proc_macro::Punct::new(',', ::proc_macro::Spacing::Alone) 110 + )); 111 + quote_spanned!(@proc $v $span $($tt)*); 112 + }; 113 + (@proc $v:ident $span:ident @ $($tt:tt)*) => { 114 + $v.push(::proc_macro::TokenTree::Punct( 115 + ::proc_macro::Punct::new('@', ::proc_macro::Spacing::Alone) 116 + )); 117 + quote_spanned!(@proc $v $span $($tt)*); 118 + }; 119 + (@proc $v:ident $span:ident ! $($tt:tt)*) => { 120 + $v.push(::proc_macro::TokenTree::Punct( 121 + ::proc_macro::Punct::new('!', ::proc_macro::Spacing::Alone) 122 + )); 123 + quote_spanned!(@proc $v $span $($tt)*); 124 + }; 125 + (@proc $v:ident $span:ident $id:ident $($tt:tt)*) => { 126 + $v.push(::proc_macro::TokenTree::Ident(::proc_macro::Ident::new(stringify!($id), $span))); 127 + quote_spanned!(@proc $v $span $($tt)*); 128 + }; 129 + } 130 + 131 + /// Converts tokens into [`proc_macro::TokenStream`] and performs variable interpolations with 132 + /// mixed site span ([`Span::mixed_site()`]). 133 + /// 134 + /// This is a similar to the [`quote!`](https://docs.rs/quote/latest/quote/macro.quote.html) macro 135 + /// from the `quote` crate but provides only just enough functionality needed by the current 136 + /// `macros` crate. 137 + /// 138 + /// [`Span::mixed_site()`]: https://doc.rust-lang.org/proc_macro/struct.Span.html#method.mixed_site 139 + macro_rules! quote { 140 + ($($tt:tt)*) => { 141 + quote_spanned!(::proc_macro::Span::mixed_site() => $($tt)*) 142 + } 143 + }

+27

rust/uapi/lib.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + //! UAPI Bindings. 4 + //! 5 + //! Contains the bindings generated by `bindgen` for UAPI interfaces. 6 + //! 7 + //! This crate may be used directly by drivers that need to interact with 8 + //! userspace APIs. 9 + 10 + #![no_std] 11 + #![feature(core_ffi_c)] 12 + // See <https://github.com/rust-lang/rust-bindgen/issues/1651>. 13 + #![cfg_attr(test, allow(deref_nullptr))] 14 + #![cfg_attr(test, allow(unaligned_references))] 15 + #![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))] 16 + #![allow( 17 + clippy::all, 18 + missing_docs, 19 + non_camel_case_types, 20 + non_upper_case_globals, 21 + non_snake_case, 22 + improper_ctypes, 23 + unreachable_pub, 24 + unsafe_op_in_unsafe_fn 25 + )] 26 + 27 + include!(concat!(env!("OBJTREE"), "/rust/uapi/uapi_generated.rs"));

+9

rust/uapi/uapi_helper.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + /* 3 + * Header that contains the headers for which Rust UAPI bindings 4 + * will be automatically generated by `bindgen`. 5 + * 6 + * Sorted alphabetically. 7 + */ 8 + 9 + #include <uapi/asm-generic/ioctl.h>

+26

samples/rust/rust_print.rs

··· 15 15 16 16 struct RustPrint; 17 17 18 + fn arc_print() -> Result { 19 + use kernel::sync::*; 20 + 21 + let a = Arc::try_new(1)?; 22 + let b = UniqueArc::try_new("hello, world")?; 23 + 24 + // Prints the value of data in `a`. 25 + pr_info!("{}", a); 26 + 27 + // Uses ":?" to print debug fmt of `b`. 28 + pr_info!("{:?}", b); 29 + 30 + let a: Arc<&str> = b.into(); 31 + let c = a.clone(); 32 + 33 + // Uses `dbg` to print, will move `c` (for temporary debugging purposes). 34 + dbg!(c); 35 + 36 + // Pretty-prints the debug formatting with lower-case hexadecimal integers. 37 + pr_info!("{:#x?}", a); 38 + 39 + Ok(()) 40 + } 41 + 18 42 impl kernel::Module for RustPrint { 19 43 fn init(_module: &'static ThisModule) -> Result<Self> { 20 44 pr_info!("Rust printing macros sample (init)\n"); ··· 66 42 pr_info!("A {} that", "line"); 67 43 pr_cont!(" is {}", "continued"); 68 44 pr_cont!(" with {}\n", "args"); 45 + 46 + arc_print()?; 69 47 70 48 Ok(RustPrint) 71 49 }

+1 -1

scripts/Makefile.build

··· 277 277 # Compile Rust sources (.rs) 278 278 # --------------------------------------------------------------------------- 279 279 280 - rust_allowed_features := core_ffi_c 280 + rust_allowed_features := core_ffi_c,explicit_generic_args_with_impl_trait,new_uninit,pin_macro 281 281 282 282 rust_common_cmd = \ 283 283 RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \

Configure Feed

Configure Feed