rust: add support for NUMA ids in allocations

+4 -4

rust/helpers/slab.c

··· 3 3 #include <linux/slab.h> 4 4 5 5 void * __must_check __realloc_size(2) 6 - rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) 6 + rust_helper_krealloc_node(const void *objp, size_t new_size, gfp_t flags, int node) 7 7 { 8 - return krealloc(objp, new_size, flags); 8 + return krealloc_node(objp, new_size, flags, node); 9 9 } 10 10 11 11 void * __must_check __realloc_size(2) 12 - rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags) 12 + rust_helper_kvrealloc_node(const void *p, size_t size, gfp_t flags, int node) 13 13 { 14 - return kvrealloc(p, size, flags); 14 + return kvrealloc_node(p, size, flags, node); 15 15 }

+2 -2

rust/helpers/vmalloc.c

··· 3 3 #include <linux/vmalloc.h> 4 4 5 5 void * __must_check __realloc_size(2) 6 - rust_helper_vrealloc(const void *p, size_t size, gfp_t flags) 6 + rust_helper_vrealloc_node(const void *p, size_t size, gfp_t flags, int node) 7 7 { 8 - return vrealloc(p, size, flags); 8 + return vrealloc_node(p, size, flags, node); 9 9 }

+49 -5

rust/kernel/alloc.rs

··· 28 28 /// Indicates an allocation error. 29 29 #[derive(Copy, Clone, PartialEq, Eq, Debug)] 30 30 pub struct AllocError; 31 + 32 + use crate::error::{code::EINVAL, Result}; 31 33 use core::{alloc::Layout, ptr::NonNull}; 32 34 33 35 /// Flags to be used when allocating memory. ··· 117 115 pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN); 118 116 } 119 117 118 + /// Non Uniform Memory Access (NUMA) node identifier. 119 + #[derive(Clone, Copy, PartialEq)] 120 + pub struct NumaNode(i32); 121 + 122 + impl NumaNode { 123 + /// Create a new NUMA node identifier (non-negative integer). 124 + /// 125 + /// Returns [`EINVAL`] if a negative id or an id exceeding [`bindings::MAX_NUMNODES`] is 126 + /// specified. 127 + pub fn new(node: i32) -> Result<Self> { 128 + // MAX_NUMNODES never exceeds 2**10 because NODES_SHIFT is 0..10. 129 + if node < 0 || node >= bindings::MAX_NUMNODES as i32 { 130 + return Err(EINVAL); 131 + } 132 + Ok(Self(node)) 133 + } 134 + } 135 + 136 + /// Specify necessary constant to pass the information to Allocator that the caller doesn't care 137 + /// about the NUMA node to allocate memory from. 138 + impl NumaNode { 139 + /// No node preference. 140 + pub const NO_NODE: NumaNode = NumaNode(bindings::NUMA_NO_NODE); 141 + } 142 + 120 143 /// The kernel's [`Allocator`] trait. 121 144 /// 122 145 /// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described ··· 164 137 /// - Implementers must ensure that all trait functions abide by the guarantees documented in the 165 138 /// `# Guarantees` sections. 166 139 pub unsafe trait Allocator { 167 - /// Allocate memory based on `layout` and `flags`. 140 + /// Allocate memory based on `layout`, `flags` and `nid`. 168 141 /// 169 142 /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout 170 143 /// constraints (i.e. minimum size and alignment as specified by `layout`). ··· 180 153 /// 181 154 /// Additionally, `Flags` are honored as documented in 182 155 /// <https://docs.kernel.org/core-api/mm-api.html#mm-api-gfp-flags>. 183 - fn alloc(layout: Layout, flags: Flags) -> Result<NonNull<[u8]>, AllocError> { 156 + fn alloc(layout: Layout, flags: Flags, nid: NumaNode) -> Result<NonNull<[u8]>, AllocError> { 184 157 // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a 185 158 // new memory allocation. 186 - unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags) } 159 + unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, nid) } 187 160 } 188 161 189 - /// Re-allocate an existing memory allocation to satisfy the requested `layout`. 162 + /// Re-allocate an existing memory allocation to satisfy the requested `layout` and 163 + /// a specific NUMA node request to allocate the memory for. 164 + /// 165 + /// Systems employing a Non Uniform Memory Access (NUMA) architecture contain collections of 166 + /// hardware resources including processors, memory, and I/O buses, that comprise what is 167 + /// commonly known as a NUMA node. 168 + /// 169 + /// `nid` stands for NUMA id, i. e. NUMA node identifier, which is a non-negative integer 170 + /// if a node needs to be specified, or [`NumaNode::NO_NODE`] if the caller doesn't care. 190 171 /// 191 172 /// If the requested size is zero, `realloc` behaves equivalent to `free`. 192 173 /// ··· 231 196 layout: Layout, 232 197 old_layout: Layout, 233 198 flags: Flags, 199 + nid: NumaNode, 234 200 ) -> Result<NonNull<[u8]>, AllocError>; 235 201 236 202 /// Free an existing memory allocation. ··· 247 211 // SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this 248 212 // allocator. We are passing a `Layout` with the smallest possible alignment, so it is 249 213 // smaller than or equal to the alignment previously used with this allocation. 250 - let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) }; 214 + let _ = unsafe { 215 + Self::realloc( 216 + Some(ptr), 217 + Layout::new::<()>(), 218 + layout, 219 + Flags(0), 220 + NumaNode::NO_NODE, 221 + ) 222 + }; 251 223 } 252 224 } 253 225

+22 -13

rust/kernel/alloc/allocator.rs

··· 13 13 use core::ptr; 14 14 use core::ptr::NonNull; 15 15 16 - use crate::alloc::{AllocError, Allocator}; 16 + use crate::alloc::{AllocError, Allocator, NumaNode}; 17 17 use crate::bindings; 18 18 use crate::pr_warn; 19 19 ··· 45 45 46 46 /// # Invariants 47 47 /// 48 - /// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. 48 + /// One of the following: `krealloc_node`, `vrealloc_node`, `kvrealloc_node`. 49 49 struct ReallocFunc( 50 - unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void, 50 + unsafe extern "C" fn( 51 + *const crate::ffi::c_void, 52 + usize, 53 + u32, 54 + crate::ffi::c_int, 55 + ) -> *mut crate::ffi::c_void, 51 56 ); 52 57 53 58 impl ReallocFunc { 54 - // INVARIANT: `krealloc` satisfies the type invariants. 55 - const KREALLOC: Self = Self(bindings::krealloc); 59 + // INVARIANT: `krealloc_node` satisfies the type invariants. 60 + const KREALLOC: Self = Self(bindings::krealloc_node); 56 61 57 - // INVARIANT: `vrealloc` satisfies the type invariants. 58 - const VREALLOC: Self = Self(bindings::vrealloc); 62 + // INVARIANT: `vrealloc_node` satisfies the type invariants. 63 + const VREALLOC: Self = Self(bindings::vrealloc_node); 59 64 60 - // INVARIANT: `kvrealloc` satisfies the type invariants. 61 - const KVREALLOC: Self = Self(bindings::kvrealloc); 65 + // INVARIANT: `kvrealloc_node` satisfies the type invariants. 66 + const KVREALLOC: Self = Self(bindings::kvrealloc_node); 62 67 63 68 /// # Safety 64 69 /// ··· 81 76 layout: Layout, 82 77 old_layout: Layout, 83 78 flags: Flags, 79 + nid: NumaNode, 84 80 ) -> Result<NonNull<[u8]>, AllocError> { 85 81 let size = layout.size(); 86 82 let ptr = match ptr { ··· 105 99 // - Those functions provide the guarantees of this function. 106 100 let raw_ptr = unsafe { 107 101 // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. 108 - self.0(ptr.cast(), size, flags.0).cast() 102 + self.0(ptr.cast(), size, flags.0, nid.0).cast() 109 103 }; 110 104 111 105 let ptr = if size == 0 { ··· 140 134 layout: Layout, 141 135 old_layout: Layout, 142 136 flags: Flags, 137 + nid: NumaNode, 143 138 ) -> Result<NonNull<[u8]>, AllocError> { 144 139 let layout = Kmalloc::aligned_layout(layout); 145 140 146 141 // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. 147 - unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } 142 + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) } 148 143 } 149 144 } 150 145 ··· 160 153 layout: Layout, 161 154 old_layout: Layout, 162 155 flags: Flags, 156 + nid: NumaNode, 163 157 ) -> Result<NonNull<[u8]>, AllocError> { 164 158 // TODO: Support alignments larger than PAGE_SIZE. 165 159 if layout.align() > bindings::PAGE_SIZE { ··· 170 162 171 163 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 172 164 // allocated with this `Allocator`. 173 - unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } 165 + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) } 174 166 } 175 167 } 176 168 ··· 185 177 layout: Layout, 186 178 old_layout: Layout, 187 179 flags: Flags, 180 + nid: NumaNode, 188 181 ) -> Result<NonNull<[u8]>, AllocError> { 189 182 // `KVmalloc` may use the `Kmalloc` backend, hence we have to enforce a `Kmalloc` 190 183 // compatible layout. ··· 199 190 200 191 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 201 192 // allocated with this `Allocator`. 202 - unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } 193 + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) } 203 194 } 204 195 }

+2 -1

rust/kernel/alloc/allocator_test.rs

··· 9 9 10 10 #![allow(missing_docs)] 11 11 12 - use super::{flags::*, AllocError, Allocator, Flags}; 12 + use super::{flags::*, AllocError, Allocator, Flags, NumaNode}; 13 13 use core::alloc::Layout; 14 14 use core::cmp; 15 15 use core::ptr; ··· 51 51 layout: Layout, 52 52 old_layout: Layout, 53 53 flags: Flags, 54 + _nid: NumaNode, 54 55 ) -> Result<NonNull<[u8]>, AllocError> { 55 56 let src = match ptr { 56 57 Some(src) => {

+2 -2

rust/kernel/alloc/kbox.rs

··· 4 4 5 5 #[allow(unused_imports)] // Used in doc comments. 6 6 use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; 7 - use super::{AllocError, Allocator, Flags}; 7 + use super::{AllocError, Allocator, Flags, NumaNode}; 8 8 use core::alloc::Layout; 9 9 use core::borrow::{Borrow, BorrowMut}; 10 10 use core::fmt; ··· 273 273 /// ``` 274 274 pub fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>, A>, AllocError> { 275 275 let layout = Layout::new::<MaybeUninit<T>>(); 276 - let ptr = A::alloc(layout, flags)?; 276 + let ptr = A::alloc(layout, flags, NumaNode::NO_NODE)?; 277 277 278 278 // INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`, 279 279 // which is sufficient in size and alignment for storing a `T`.

+9 -2

rust/kernel/alloc/kvec.rs

··· 5 5 use super::{ 6 6 allocator::{KVmalloc, Kmalloc, Vmalloc}, 7 7 layout::ArrayLayout, 8 - AllocError, Allocator, Box, Flags, 8 + AllocError, Allocator, Box, Flags, NumaNode, 9 9 }; 10 10 use core::{ 11 11 borrow::{Borrow, BorrowMut}, ··· 634 634 layout.into(), 635 635 self.layout.into(), 636 636 flags, 637 + NumaNode::NO_NODE, 637 638 )? 638 639 }; 639 640 ··· 1112 1111 // the type invariant to be smaller than `cap`. Depending on `realloc` this operation 1113 1112 // may shrink the buffer or leave it as it is. 1114 1113 ptr = match unsafe { 1115 - A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags) 1114 + A::realloc( 1115 + Some(buf.cast()), 1116 + layout.into(), 1117 + old_layout.into(), 1118 + flags, 1119 + NumaNode::NO_NODE, 1120 + ) 1116 1121 } { 1117 1122 // If we fail to shrink, which likely can't even happen, continue with the existing 1118 1123 // buffer.

Configure Feed

Configure Feed