tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm, slab: don't wrap internal functions with alloc_hooks()

The functions __kmalloc_noprof(), kmalloc_large_noprof(),
kmalloc_trace_noprof() and their _node variants are all internal to the
implementations of kmalloc_noprof() and kmalloc_node_noprof() and are
only declared in the "public" slab.h and exported so that those
implementations can be static inline and distinguish the build-time
constant size variants. The only other users for some of the internal
functions are slub_kunit and fortify_kunit tests which make very
short-lived allocations.

Therefore we can stop wrapping them with the alloc_hooks() macro.
Instead add a __ prefix to all of them and a comment documenting these
as internal. Also rename __kmalloc_trace() to __kmalloc_cache() which is
more descriptive - it is a variant of __kmalloc() where the exact
kmalloc cache has been already determined.

The usage in fortify_kunit can be removed completely, as the internal
functions should be tested already through kmalloc() tests in the
test variant that passes non-constant allocation size.

Reported-by: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Kees Cook <keescook@chromium.org>
Reviewed-by: Kent Overstreet <kent.overstreet@linux.dev>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

Vlastimil Babka a0a44d91 1613e604

+38 -43
+24 -24
include/linux/slab.h
··· 528 528 529 529 #include <linux/alloc_tag.h> 530 530 531 - void *__kmalloc_noprof(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1); 532 - #define __kmalloc(...) alloc_hooks(__kmalloc_noprof(__VA_ARGS__)) 533 - 534 531 /** 535 532 * kmem_cache_alloc - Allocate an object 536 533 * @cachep: The cache to allocate from. ··· 565 568 kmem_cache_free_bulk(NULL, size, p); 566 569 } 567 570 568 - void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment 569 - __alloc_size(1); 570 - #define __kmalloc_node(...) alloc_hooks(__kmalloc_node_noprof(__VA_ARGS__)) 571 - 572 571 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags, 573 572 int node) __assume_slab_alignment __malloc; 574 573 #define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__)) 575 574 576 - void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t flags, size_t size) 577 - __assume_kmalloc_alignment __alloc_size(3); 575 + /* 576 + * The following functions are not to be used directly and are intended only 577 + * for internal use from kmalloc() and kmalloc_node() 578 + * with the exception of kunit tests 579 + */ 578 580 579 - void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, 580 - int node, size_t size) __assume_kmalloc_alignment 581 - __alloc_size(4); 582 - #define kmalloc_trace(...) alloc_hooks(kmalloc_trace_noprof(__VA_ARGS__)) 581 + void *__kmalloc_noprof(size_t size, gfp_t flags) 582 + __assume_kmalloc_alignment __alloc_size(1); 583 583 584 - #define kmalloc_node_trace(...) alloc_hooks(kmalloc_node_trace_noprof(__VA_ARGS__)) 584 + void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) 585 + __assume_kmalloc_alignment __alloc_size(1); 585 586 586 - void *kmalloc_large_noprof(size_t size, gfp_t flags) __assume_page_alignment 587 - __alloc_size(1); 588 - #define kmalloc_large(...) alloc_hooks(kmalloc_large_noprof(__VA_ARGS__)) 587 + void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size) 588 + __assume_kmalloc_alignment __alloc_size(3); 589 589 590 - void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) __assume_page_alignment 591 - __alloc_size(1); 592 - #define kmalloc_large_node(...) alloc_hooks(kmalloc_large_node_noprof(__VA_ARGS__)) 590 + void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, 591 + int node, size_t size) 592 + __assume_kmalloc_alignment __alloc_size(4); 593 + 594 + void *__kmalloc_large_noprof(size_t size, gfp_t flags) 595 + __assume_page_alignment __alloc_size(1); 596 + 597 + void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) 598 + __assume_page_alignment __alloc_size(1); 593 599 594 600 /** 595 601 * kmalloc - allocate kernel memory ··· 654 654 unsigned int index; 655 655 656 656 if (size > KMALLOC_MAX_CACHE_SIZE) 657 - return kmalloc_large_noprof(size, flags); 657 + return __kmalloc_large_noprof(size, flags); 658 658 659 659 index = kmalloc_index(size); 660 - return kmalloc_trace_noprof( 660 + return __kmalloc_cache_noprof( 661 661 kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], 662 662 flags, size); 663 663 } ··· 671 671 unsigned int index; 672 672 673 673 if (size > KMALLOC_MAX_CACHE_SIZE) 674 - return kmalloc_large_node_noprof(size, flags, node); 674 + return __kmalloc_large_node_noprof(size, flags, node); 675 675 676 676 index = kmalloc_index(size); 677 - return kmalloc_node_trace_noprof( 677 + return __kmalloc_cache_node_noprof( 678 678 kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], 679 679 flags, node, size); 680 680 }
-5
lib/fortify_kunit.c
··· 234 234 checker(expected_size, \ 235 235 kmalloc_array_node(alloc_size, 1, gfp, NUMA_NO_NODE), \ 236 236 kfree(p)); \ 237 - checker(expected_size, __kmalloc(alloc_size, gfp), \ 238 - kfree(p)); \ 239 - checker(expected_size, \ 240 - __kmalloc_node(alloc_size, gfp, NUMA_NO_NODE), \ 241 - kfree(p)); \ 242 237 \ 243 238 orig = kmalloc(alloc_size, gfp); \ 244 239 KUNIT_EXPECT_TRUE(test, orig != NULL); \
+1 -1
lib/slub_kunit.c
··· 140 140 { 141 141 struct kmem_cache *s = test_kmem_cache_create("TestSlub_RZ_kmalloc", 32, 142 142 SLAB_KMALLOC|SLAB_STORE_USER|SLAB_RED_ZONE); 143 - u8 *p = kmalloc_trace(s, GFP_KERNEL, 18); 143 + u8 *p = __kmalloc_cache_noprof(s, GFP_KERNEL, 18); 144 144 145 145 kasan_disable_current(); 146 146
+13 -13
mm/slub.c
··· 4053 4053 * directly to the page allocator. We use __GFP_COMP, because we will need to 4054 4054 * know the allocation order to free the pages properly in kfree. 4055 4055 */ 4056 - static void *__kmalloc_large_node(size_t size, gfp_t flags, int node) 4056 + static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) 4057 4057 { 4058 4058 struct folio *folio; 4059 4059 void *ptr = NULL; ··· 4078 4078 return ptr; 4079 4079 } 4080 4080 4081 - void *kmalloc_large_noprof(size_t size, gfp_t flags) 4081 + void *__kmalloc_large_noprof(size_t size, gfp_t flags) 4082 4082 { 4083 - void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE); 4083 + void *ret = ___kmalloc_large_node(size, flags, NUMA_NO_NODE); 4084 4084 4085 4085 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), 4086 4086 flags, NUMA_NO_NODE); 4087 4087 return ret; 4088 4088 } 4089 - EXPORT_SYMBOL(kmalloc_large_noprof); 4089 + EXPORT_SYMBOL(__kmalloc_large_noprof); 4090 4090 4091 - void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) 4091 + void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) 4092 4092 { 4093 - void *ret = __kmalloc_large_node(size, flags, node); 4093 + void *ret = ___kmalloc_large_node(size, flags, node); 4094 4094 4095 4095 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), 4096 4096 flags, node); 4097 4097 return ret; 4098 4098 } 4099 - EXPORT_SYMBOL(kmalloc_large_node_noprof); 4099 + EXPORT_SYMBOL(__kmalloc_large_node_noprof); 4100 4100 4101 4101 static __always_inline 4102 4102 void *__do_kmalloc_node(size_t size, gfp_t flags, int node, ··· 4106 4106 void *ret; 4107 4107 4108 4108 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 4109 - ret = __kmalloc_large_node(size, flags, node); 4109 + ret = __kmalloc_large_node_noprof(size, flags, node); 4110 4110 trace_kmalloc(caller, ret, size, 4111 4111 PAGE_SIZE << get_order(size), flags, node); 4112 4112 return ret; ··· 4142 4142 } 4143 4143 EXPORT_SYMBOL(kmalloc_node_track_caller_noprof); 4144 4144 4145 - void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) 4145 + void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) 4146 4146 { 4147 4147 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, 4148 4148 _RET_IP_, size); ··· 4152 4152 ret = kasan_kmalloc(s, ret, size, gfpflags); 4153 4153 return ret; 4154 4154 } 4155 - EXPORT_SYMBOL(kmalloc_trace_noprof); 4155 + EXPORT_SYMBOL(__kmalloc_cache_noprof); 4156 4156 4157 - void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, 4158 - int node, size_t size) 4157 + void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, 4158 + int node, size_t size) 4159 4159 { 4160 4160 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); 4161 4161 ··· 4164 4164 ret = kasan_kmalloc(s, ret, size, gfpflags); 4165 4165 return ret; 4166 4166 } 4167 - EXPORT_SYMBOL(kmalloc_node_trace_noprof); 4167 + EXPORT_SYMBOL(__kmalloc_cache_node_noprof); 4168 4168 4169 4169 static noinline void free_to_partial_list( 4170 4170 struct kmem_cache *s, struct slab *slab,