tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm/slub: allow to set node and align in k[v]realloc

Reimplement k[v]realloc_node() to be able to set node and alignment should
a user need to do so. In order to do that while retaining the maximal
backward compatibility, add k[v]realloc_node_align() functions and
redefine the rest of API using these new ones.

While doing that, we also keep the number of _noprof variants to a
minimum, which implies some changes to the existing users of older _noprof
functions, that basically being bcachefs.

With that change we also provide the ability for the Rust part of the
kernel to set node and alignment in its K[v]xxx [re]allocations.

Link: https://lkml.kernel.org/r/20250806124147.1724658-1-vitaly.wool@konsulko.se
Signed-off-by: Vitaly Wool <vitaly.wool@konsulko.se>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Danilo Krummrich <dakr@kernel.org>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Jann Horn <jannh@google.com>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Vitaly Wool and committed by
Andrew Morton 2cd82317 4c5d3365

+74 -34
+1 -1
fs/bcachefs/darray.c
··· 21 21 return -ENOMEM; 22 22 23 23 void *data = likely(bytes < INT_MAX) 24 - ? kvmalloc_noprof(bytes, gfp) 24 + ? kvmalloc_node_align_noprof(bytes, 1, gfp, NUMA_NO_NODE) 25 25 : vmalloc_noprof(bytes); 26 26 if (!data) 27 27 return -ENOMEM;
+1 -1
fs/bcachefs/util.h
··· 61 61 { 62 62 void *p = unlikely(n >= INT_MAX) 63 63 ? vmalloc_noprof(n) 64 - : kvmalloc_noprof(n, flags & ~__GFP_ZERO); 64 + : kvmalloc_node_align_noprof(n, 1, flags & ~__GFP_ZERO, NUMA_NO_NODE); 65 65 if (p && (flags & __GFP_ZERO)) 66 66 memset(p, 0, n); 67 67 return p;
+1 -1
include/linux/bpfptr.h
··· 67 67 68 68 static inline void *kvmemdup_bpfptr_noprof(bpfptr_t src, size_t len) 69 69 { 70 - void *p = kvmalloc_noprof(len, GFP_USER | __GFP_NOWARN); 70 + void *p = kvmalloc_node_align_noprof(len, 1, GFP_USER | __GFP_NOWARN, NUMA_NO_NODE); 71 71 72 72 if (!p) 73 73 return ERR_PTR(-ENOMEM);
+24 -15
include/linux/slab.h
··· 465 465 /* 466 466 * Common kmalloc functions provided by all allocators 467 467 */ 468 - void * __must_check krealloc_noprof(const void *objp, size_t new_size, 469 - gfp_t flags) __realloc_size(2); 470 - #define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__)) 468 + void * __must_check krealloc_node_align_noprof(const void *objp, size_t new_size, 469 + unsigned long align, 470 + gfp_t flags, int nid) __realloc_size(2); 471 + #define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, _s, 1, _f, NUMA_NO_NODE) 472 + #define krealloc_node_align(...) alloc_hooks(krealloc_node_align_noprof(__VA_ARGS__)) 473 + #define krealloc_node(_o, _s, _f, _n) krealloc_node_align(_o, _s, 1, _f, _n) 474 + #define krealloc(...) krealloc_node(__VA_ARGS__, NUMA_NO_NODE) 471 475 472 476 void kfree(const void *objp); 473 477 void kfree_sensitive(const void *objp); ··· 1045 1041 #define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__)) 1046 1042 #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node) 1047 1043 1048 - void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) __alloc_size(1); 1049 - #define kvmalloc_node_noprof(size, flags, node) \ 1050 - __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node) 1051 - #define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) 1052 - 1053 - #define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE) 1054 - #define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE) 1044 + void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align, 1045 + gfp_t flags, int node) __alloc_size(1); 1046 + #define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \ 1047 + __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _node) 1048 + #define kvmalloc_node_align(...) \ 1049 + alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__)) 1050 + #define kvmalloc_node(_s, _f, _n) kvmalloc_node_align(_s, 1, _f, _n) 1051 + #define kvmalloc(...) kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE) 1055 1052 #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO) 1056 1053 1057 1054 #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node) 1055 + 1058 1056 #define kmem_buckets_valloc(_b, _size, _flags) \ 1059 - alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE)) 1057 + alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _flags, NUMA_NO_NODE)) 1060 1058 1061 1059 static inline __alloc_size(1, 2) void * 1062 1060 kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) ··· 1068 1062 if (unlikely(check_mul_overflow(n, size, &bytes))) 1069 1063 return NULL; 1070 1064 1071 - return kvmalloc_node_noprof(bytes, flags, node); 1065 + return kvmalloc_node_align_noprof(bytes, 1, flags, node); 1072 1066 } 1073 1067 1074 1068 #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE) ··· 1079 1073 #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__)) 1080 1074 #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__)) 1081 1075 1082 - void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) 1083 - __realloc_size(2); 1084 - #define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__)) 1076 + void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align, 1077 + gfp_t flags, int nid) __realloc_size(2); 1078 + #define kvrealloc_node_align(...) \ 1079 + alloc_hooks(kvrealloc_node_align_noprof(__VA_ARGS__)) 1080 + #define kvrealloc_node(_p, _s, _f, _n) kvrealloc_node_align(_p, _s, 1, _f, _n) 1081 + #define kvrealloc(...) kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE) 1085 1082 1086 1083 extern void kvfree(const void *addr); 1087 1084 DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T))
+2 -2
lib/rhashtable.c
··· 184 184 static struct lock_class_key __key; 185 185 186 186 tbl = alloc_hooks_tag(ht->alloc_tag, 187 - kvmalloc_node_noprof(struct_size(tbl, buckets, nbuckets), 188 - gfp|__GFP_ZERO, NUMA_NO_NODE)); 187 + kvmalloc_node_align_noprof(struct_size(tbl, buckets, nbuckets), 188 + 1, gfp|__GFP_ZERO, NUMA_NO_NODE)); 189 189 190 190 size = nbuckets; 191 191
+45 -14
mm/slub.c
··· 4881 4881 EXPORT_SYMBOL(kfree); 4882 4882 4883 4883 static __always_inline __realloc_size(2) void * 4884 - __do_krealloc(const void *p, size_t new_size, gfp_t flags) 4884 + __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid) 4885 4885 { 4886 4886 void *ret; 4887 4887 size_t ks = 0; ··· 4894 4894 /* Check for double-free. */ 4895 4895 if (!kasan_check_byte(p)) 4896 4896 return NULL; 4897 + 4898 + /* 4899 + * If reallocation is not necessary (e. g. the new size is less 4900 + * than the current allocated size), the current allocation will be 4901 + * preserved unless __GFP_THISNODE is set. In the latter case a new 4902 + * allocation on the requested node will be attempted. 4903 + */ 4904 + if (unlikely(flags & __GFP_THISNODE) && nid != NUMA_NO_NODE && 4905 + nid != page_to_nid(virt_to_page(p))) 4906 + goto alloc_new; 4897 4907 4898 4908 if (is_kfence_address(p)) { 4899 4909 ks = orig_size = kfence_ksize(p); ··· 4925 4915 4926 4916 /* If the old object doesn't fit, allocate a bigger one */ 4927 4917 if (new_size > ks) 4918 + goto alloc_new; 4919 + 4920 + /* If the old object doesn't satisfy the new alignment, allocate a new one */ 4921 + if (!IS_ALIGNED((unsigned long)p, align)) 4928 4922 goto alloc_new; 4929 4923 4930 4924 /* Zero out spare memory. */ ··· 4953 4939 return (void *)p; 4954 4940 4955 4941 alloc_new: 4956 - ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_); 4942 + ret = kmalloc_node_track_caller_noprof(new_size, flags, nid, _RET_IP_); 4957 4943 if (ret && p) { 4958 4944 /* Disable KASAN checks as the object's redzone is accessed. */ 4959 4945 kasan_disable_current(); ··· 4965 4951 } 4966 4952 4967 4953 /** 4968 - * krealloc - reallocate memory. The contents will remain unchanged. 4954 + * krealloc_node_align - reallocate memory. The contents will remain unchanged. 4969 4955 * @p: object to reallocate memory for. 4970 4956 * @new_size: how many bytes of memory are required. 4957 + * @align: desired alignment. 4971 4958 * @flags: the type of memory to allocate. 4959 + * @nid: NUMA node or NUMA_NO_NODE 4972 4960 * 4973 4961 * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size 4974 4962 * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 4963 + * 4964 + * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 4965 + * Documentation/core-api/memory-allocation.rst for more details. 4975 4966 * 4976 4967 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 4977 4968 * initial memory allocation, every subsequent call to this API for the same ··· 5002 4983 * 5003 4984 * Return: pointer to the allocated memory or %NULL in case of error 5004 4985 */ 5005 - void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags) 4986 + void *krealloc_node_align_noprof(const void *p, size_t new_size, unsigned long align, 4987 + gfp_t flags, int nid) 5006 4988 { 5007 4989 void *ret; 5008 4990 ··· 5012 4992 return ZERO_SIZE_PTR; 5013 4993 } 5014 4994 5015 - ret = __do_krealloc(p, new_size, flags); 4995 + ret = __do_krealloc(p, new_size, align, flags, nid); 5016 4996 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret)) 5017 4997 kfree(p); 5018 4998 5019 4999 return ret; 5020 5000 } 5021 - EXPORT_SYMBOL(krealloc_noprof); 5001 + EXPORT_SYMBOL(krealloc_node_align_noprof); 5022 5002 5023 5003 static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size) 5024 5004 { ··· 5049 5029 * failure, fall back to non-contiguous (vmalloc) allocation. 5050 5030 * @size: size of the request. 5051 5031 * @b: which set of kmalloc buckets to allocate from. 5032 + * @align: desired alignment. 5052 5033 * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. 5053 5034 * @node: numa node to allocate from 5035 + * 5036 + * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 5037 + * Documentation/core-api/memory-allocation.rst for more details. 5054 5038 * 5055 5039 * Uses kmalloc to get the memory but if the allocation fails then falls back 5056 5040 * to the vmalloc allocator. Use kvfree for freeing the memory. ··· 5065 5041 * 5066 5042 * Return: pointer to the allocated memory of %NULL in case of failure 5067 5043 */ 5068 - void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) 5044 + void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align, 5045 + gfp_t flags, int node) 5069 5046 { 5070 5047 void *ret; 5071 5048 ··· 5096 5071 * about the resulting pointer, and cannot play 5097 5072 * protection games. 5098 5073 */ 5099 - return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, 5074 + return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_END, 5100 5075 flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, 5101 5076 node, __builtin_return_address(0)); 5102 5077 } ··· 5140 5115 EXPORT_SYMBOL(kvfree_sensitive); 5141 5116 5142 5117 /** 5143 - * kvrealloc - reallocate memory; contents remain unchanged 5118 + * kvrealloc_node_align - reallocate memory; contents remain unchanged 5144 5119 * @p: object to reallocate memory for 5145 5120 * @size: the size to reallocate 5121 + * @align: desired alignment 5146 5122 * @flags: the flags for the page level allocator 5123 + * @nid: NUMA node id 5147 5124 * 5148 5125 * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0 5149 5126 * and @p is not a %NULL pointer, the object pointed to is freed. 5127 + * 5128 + * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 5129 + * Documentation/core-api/memory-allocation.rst for more details. 5150 5130 * 5151 5131 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 5152 5132 * initial memory allocation, every subsequent call to this API for the same ··· 5166 5136 * 5167 5137 * Return: pointer to the allocated memory or %NULL in case of error 5168 5138 */ 5169 - void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) 5139 + void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align, 5140 + gfp_t flags, int nid) 5170 5141 { 5171 5142 void *n; 5172 5143 5173 5144 if (is_vmalloc_addr(p)) 5174 - return vrealloc_noprof(p, size, flags); 5145 + return vrealloc_node_align_noprof(p, size, align, flags, nid); 5175 5146 5176 - n = krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size)); 5147 + n = krealloc_node_align_noprof(p, size, align, kmalloc_gfp_adjust(flags, size), nid); 5177 5148 if (!n) { 5178 5149 /* We failed to krealloc(), fall back to kvmalloc(). */ 5179 - n = kvmalloc_noprof(size, flags); 5150 + n = kvmalloc_node_align_noprof(size, align, flags, nid); 5180 5151 if (!n) 5181 5152 return NULL; 5182 5153 ··· 5193 5162 5194 5163 return n; 5195 5164 } 5196 - EXPORT_SYMBOL(kvrealloc_noprof); 5165 + EXPORT_SYMBOL(kvrealloc_node_align_noprof); 5197 5166 5198 5167 struct detached_freelist { 5199 5168 struct slab *slab;