tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'slab-for-6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab

Pull slab updates from Vlastimil Babka:

- Move the TINY_RCU kvfree_rcu() implementation from RCU to SLAB
subsystem and cleanup its integration (Vlastimil Babka)

Following the move of the TREE_RCU batching kvfree_rcu()
implementation in 6.14, move also the simpler TINY_RCU variant.
Refactor the #ifdef guards so that the simple implementation is also
used with SLUB_TINY.

Remove the need for RCU to recognize fake callback function pointers
(__is_kvfree_rcu_offset()) when handling call_rcu() by implementing a
callback that calculates the object's address from the embedded
rcu_head address without knowing its offset.

- Improve kmalloc cache randomization in kvmalloc (GONG Ruiqi)

Due to an extra layer of function call, all kvmalloc() allocations
used the same set of random caches. Thanks to moving the kvmalloc()
implementation to slub.c, this is improved and randomization now
works for kvmalloc.

- Various improvements to debugging, testing and other cleanups (Hyesoo
Yu, Lilith Gkini, Uladzislau Rezki, Matthew Wilcox, Kevin Brodsky, Ye
Bin)

* tag 'slab-for-6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
slub: Handle freelist cycle in on_freelist()
mm/slab: call kmalloc_noprof() unconditionally in kmalloc_array_noprof()
slab: Mark large folios for debugging purposes
kunit, slub: Add test_kfree_rcu_wq_destroy use case
mm, slab: cleanup slab_bug() parameters
mm: slub: call WARN() when detecting a slab corruption
mm: slub: Print the broken data before restoring them
slab: Achieve better kmalloc caches randomization in kvmalloc
slab: Adjust placement of __kvmalloc_node_noprof
mm/slab: simplify SLAB_* flag handling
slab: don't batch kvfree_rcu() with SLUB_TINY
rcu, slab: use a regular callback function for kvfree_rcu
rcu: remove trace_rcu_kvfree_callback
slab, rcu: move TINY_RCU variant of kvfree_rcu() to SLAB

Linus Torvalds 05b00ffd 95c61e1a

+430 -383
+10 -8
include/linux/page-flags.h
··· 925 925 enum pagetype { 926 926 /* 0x00-0x7f are positive numbers, ie mapcount */ 927 927 /* Reserve 0x80-0xef for mapcount overflow. */ 928 - PGTY_buddy = 0xf0, 929 - PGTY_offline = 0xf1, 930 - PGTY_table = 0xf2, 931 - PGTY_guard = 0xf3, 932 - PGTY_hugetlb = 0xf4, 933 - PGTY_slab = 0xf5, 934 - PGTY_zsmalloc = 0xf6, 935 - PGTY_unaccepted = 0xf7, 928 + PGTY_buddy = 0xf0, 929 + PGTY_offline = 0xf1, 930 + PGTY_table = 0xf2, 931 + PGTY_guard = 0xf3, 932 + PGTY_hugetlb = 0xf4, 933 + PGTY_slab = 0xf5, 934 + PGTY_zsmalloc = 0xf6, 935 + PGTY_unaccepted = 0xf7, 936 + PGTY_large_kmalloc = 0xf8, 936 937 937 938 PGTY_mapcount_underflow = 0xff 938 939 }; ··· 1076 1075 * Serialized with zone lock. 1077 1076 */ 1078 1077 PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted) 1078 + FOLIO_TYPE_OPS(large_kmalloc, large_kmalloc) 1079 1079 1080 1080 /** 1081 1081 * PageHuge - Determine if the page belongs to hugetlbfs
+18 -15
include/linux/rcupdate.h
··· 1025 1025 #define RCU_POINTER_INITIALIZER(p, v) \ 1026 1026 .p = RCU_INITIALIZER(v) 1027 1027 1028 - /* 1029 - * Does the specified offset indicate that the corresponding rcu_head 1030 - * structure can be handled by kvfree_rcu()? 1031 - */ 1032 - #define __is_kvfree_rcu_offset(offset) ((offset) < 4096) 1033 - 1034 1028 /** 1035 1029 * kfree_rcu() - kfree an object after a grace period. 1036 1030 * @ptr: pointer to kfree for double-argument invocations. ··· 1035 1041 * when they are used in a kernel module, that module must invoke the 1036 1042 * high-latency rcu_barrier() function at module-unload time. 1037 1043 * 1038 - * The kfree_rcu() function handles this issue. Rather than encoding a 1039 - * function address in the embedded rcu_head structure, kfree_rcu() instead 1040 - * encodes the offset of the rcu_head structure within the base structure. 1041 - * Because the functions are not allowed in the low-order 4096 bytes of 1042 - * kernel virtual memory, offsets up to 4095 bytes can be accommodated. 1044 + * The kfree_rcu() function handles this issue. In order to have a universal 1045 + * callback function handling different offsets of rcu_head, the callback needs 1046 + * to determine the starting address of the freed object, which can be a large 1047 + * kmalloc or vmalloc allocation. To allow simply aligning the pointer down to 1048 + * page boundary for those, only offsets up to 4095 bytes can be accommodated. 1043 1049 * If the offset is larger than 4095 bytes, a compile-time error will 1044 1050 * be generated in kvfree_rcu_arg_2(). If this error is triggered, you can 1045 1051 * either fall back to use of call_rcu() or rearrange the structure to ··· 1076 1082 #define kfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr) 1077 1083 #define kvfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr) 1078 1084 1085 + /* 1086 + * In mm/slab_common.c, no suitable header to include here. 1087 + */ 1088 + void kvfree_call_rcu(struct rcu_head *head, void *ptr); 1089 + 1090 + /* 1091 + * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the 1092 + * comment of kfree_rcu() for details. 1093 + */ 1079 1094 #define kvfree_rcu_arg_2(ptr, rhf) \ 1080 1095 do { \ 1081 1096 typeof (ptr) ___p = (ptr); \ 1082 1097 \ 1083 - if (___p) { \ 1084 - BUILD_BUG_ON(!__is_kvfree_rcu_offset(offsetof(typeof(*(ptr)), rhf))); \ 1085 - kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \ 1086 - } \ 1098 + if (___p) { \ 1099 + BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096); \ 1100 + kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \ 1101 + } \ 1087 1102 } while (0) 1088 1103 1089 1104 #define kvfree_rcu_arg_1(ptr) \
-36
include/linux/rcutiny.h
··· 90 90 synchronize_rcu(); 91 91 } 92 92 93 - /* 94 - * Add one more declaration of kvfree() here. It is 95 - * not so straight forward to just include <linux/mm.h> 96 - * where it is defined due to getting many compile 97 - * errors caused by that include. 98 - */ 99 - extern void kvfree(const void *addr); 100 - 101 - static inline void __kvfree_call_rcu(struct rcu_head *head, void *ptr) 102 - { 103 - if (head) { 104 - call_rcu(head, (rcu_callback_t) ((void *) head - ptr)); 105 - return; 106 - } 107 - 108 - // kvfree_rcu(one_arg) call. 109 - might_sleep(); 110 - synchronize_rcu(); 111 - kvfree(ptr); 112 - } 113 - 114 - static inline void kvfree_rcu_barrier(void) 115 - { 116 - rcu_barrier(); 117 - } 118 - 119 - #ifdef CONFIG_KASAN_GENERIC 120 - void kvfree_call_rcu(struct rcu_head *head, void *ptr); 121 - #else 122 - static inline void kvfree_call_rcu(struct rcu_head *head, void *ptr) 123 - { 124 - __kvfree_call_rcu(head, ptr); 125 - } 126 - #endif 127 - 128 93 void rcu_qs(void); 129 94 130 95 static inline void rcu_softirq_qs(void) ··· 129 164 static inline bool rcu_inkernel_boot_has_ended(void) { return true; } 130 165 static inline bool rcu_is_watching(void) { return true; } 131 166 static inline void rcu_momentary_eqs(void) { } 132 - static inline void kfree_rcu_scheduler_running(void) { } 133 167 134 168 /* Avoid RCU read-side critical sections leaking across. */ 135 169 static inline void rcu_all_qs(void) { barrier(); }
-3
include/linux/rcutree.h
··· 34 34 } 35 35 36 36 void synchronize_rcu_expedited(void); 37 - void kvfree_call_rcu(struct rcu_head *head, void *ptr); 38 - void kvfree_rcu_barrier(void); 39 37 40 38 void rcu_barrier(void); 41 39 void rcu_momentary_eqs(void); 42 - void kfree_rcu_scheduler_running(void); 43 40 44 41 struct rcu_gp_oldstate { 45 42 unsigned long rgos_norm;
+14 -2
include/linux/slab.h
··· 16 16 #include <linux/gfp.h> 17 17 #include <linux/overflow.h> 18 18 #include <linux/types.h> 19 + #include <linux/rcupdate.h> 19 20 #include <linux/workqueue.h> 20 21 #include <linux/percpu-refcount.h> 21 22 #include <linux/cleanup.h> ··· 942 941 943 942 if (unlikely(check_mul_overflow(n, size, &bytes))) 944 943 return NULL; 945 - if (__builtin_constant_p(n) && __builtin_constant_p(size)) 946 - return kmalloc_noprof(bytes, flags); 947 944 return kmalloc_noprof(bytes, flags); 948 945 } 949 946 #define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__)) ··· 1080 1081 extern void kvfree_sensitive(const void *addr, size_t len); 1081 1082 1082 1083 unsigned int kmem_cache_size(struct kmem_cache *s); 1084 + 1085 + #ifndef CONFIG_KVFREE_RCU_BATCHED 1086 + static inline void kvfree_rcu_barrier(void) 1087 + { 1088 + rcu_barrier(); 1089 + } 1090 + 1091 + static inline void kfree_rcu_scheduler_running(void) { } 1092 + #else 1093 + void kvfree_rcu_barrier(void); 1094 + 1095 + void kfree_rcu_scheduler_running(void); 1096 + #endif 1083 1097 1084 1098 /** 1085 1099 * kmalloc_size_roundup - Report allocation bucket size for the given size
-34
include/trace/events/rcu.h
··· 561 561 ); 562 562 563 563 /* 564 - * Tracepoint for the registration of a single RCU callback of the special 565 - * kvfree() form. The first argument is the RCU type, the second argument 566 - * is a pointer to the RCU callback, the third argument is the offset 567 - * of the callback within the enclosing RCU-protected data structure, 568 - * the fourth argument is the number of lazy callbacks queued, and the 569 - * fifth argument is the total number of callbacks queued. 570 - */ 571 - TRACE_EVENT_RCU(rcu_kvfree_callback, 572 - 573 - TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset, 574 - long qlen), 575 - 576 - TP_ARGS(rcuname, rhp, offset, qlen), 577 - 578 - TP_STRUCT__entry( 579 - __field(const char *, rcuname) 580 - __field(void *, rhp) 581 - __field(unsigned long, offset) 582 - __field(long, qlen) 583 - ), 584 - 585 - TP_fast_assign( 586 - __entry->rcuname = rcuname; 587 - __entry->rhp = rhp; 588 - __entry->offset = offset; 589 - __entry->qlen = qlen; 590 - ), 591 - 592 - TP_printk("%s rhp=%p func=%ld %ld", 593 - __entry->rcuname, __entry->rhp, __entry->offset, 594 - __entry->qlen) 595 - ); 596 - 597 - /* 598 564 * Tracepoint for marking the beginning rcu_do_batch, performed to start 599 565 * RCU callback invocation. The first argument is the RCU flavor, 600 566 * the second is the number of lazy callbacks queued, the third is
-25
kernel/rcu/tiny.c
··· 85 85 static inline bool rcu_reclaim_tiny(struct rcu_head *head) 86 86 { 87 87 rcu_callback_t f; 88 - unsigned long offset = (unsigned long)head->func; 89 88 90 89 rcu_lock_acquire(&rcu_callback_map); 91 - if (__is_kvfree_rcu_offset(offset)) { 92 - trace_rcu_invoke_kvfree_callback("", head, offset); 93 - kvfree((void *)head - offset); 94 - rcu_lock_release(&rcu_callback_map); 95 - return true; 96 - } 97 90 98 91 trace_rcu_invoke_callback("", head); 99 92 f = head->func; ··· 152 159 } 153 160 EXPORT_SYMBOL_GPL(synchronize_rcu); 154 161 155 - static void tiny_rcu_leak_callback(struct rcu_head *rhp) 156 - { 157 - } 158 - 159 162 /* 160 163 * Post an RCU callback to be invoked after the end of an RCU grace 161 164 * period. But since we have but one CPU, that would be after any ··· 167 178 pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); 168 179 mem_dump_obj(head); 169 180 } 170 - 171 - if (!__is_kvfree_rcu_offset((unsigned long)head->func)) 172 - WRITE_ONCE(head->func, tiny_rcu_leak_callback); 173 181 return; 174 182 } 175 183 ··· 231 245 return oldstate == RCU_GET_STATE_COMPLETED || READ_ONCE(rcu_ctrlblk.gp_seq) != oldstate; 232 246 } 233 247 EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); 234 - 235 - #ifdef CONFIG_KASAN_GENERIC 236 - void kvfree_call_rcu(struct rcu_head *head, void *ptr) 237 - { 238 - if (head) 239 - kasan_record_aux_stack(ptr); 240 - 241 - __kvfree_call_rcu(head, ptr); 242 - } 243 - EXPORT_SYMBOL_GPL(kvfree_call_rcu); 244 - #endif 245 248 246 249 void __init rcu_init(void) 247 250 {
+2 -7
kernel/rcu/tree.c
··· 2931 2931 static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func) 2932 2932 { 2933 2933 rcu_segcblist_enqueue(&rdp->cblist, head); 2934 - if (__is_kvfree_rcu_offset((unsigned long)func)) 2935 - trace_rcu_kvfree_callback(rcu_state.name, head, 2936 - (unsigned long)func, 2937 - rcu_segcblist_n_cbs(&rdp->cblist)); 2938 - else 2939 - trace_rcu_callback(rcu_state.name, head, 2940 - rcu_segcblist_n_cbs(&rdp->cblist)); 2934 + trace_rcu_callback(rcu_state.name, head, 2935 + rcu_segcblist_n_cbs(&rdp->cblist)); 2941 2936 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued")); 2942 2937 } 2943 2938
+59
lib/tests/slub_kunit.c
··· 6 6 #include <linux/module.h> 7 7 #include <linux/kernel.h> 8 8 #include <linux/rcupdate.h> 9 + #include <linux/delay.h> 9 10 #include "../mm/slab.h" 10 11 11 12 static struct kunit_resource resource; ··· 182 181 KUNIT_EXPECT_EQ(test, 0, slab_errors); 183 182 } 184 183 184 + struct cache_destroy_work { 185 + struct work_struct work; 186 + struct kmem_cache *s; 187 + }; 188 + 189 + static void cache_destroy_workfn(struct work_struct *w) 190 + { 191 + struct cache_destroy_work *cdw; 192 + 193 + cdw = container_of(w, struct cache_destroy_work, work); 194 + kmem_cache_destroy(cdw->s); 195 + } 196 + 197 + #define KMEM_CACHE_DESTROY_NR 10 198 + 199 + static void test_kfree_rcu_wq_destroy(struct kunit *test) 200 + { 201 + struct test_kfree_rcu_struct *p; 202 + struct cache_destroy_work cdw; 203 + struct workqueue_struct *wq; 204 + struct kmem_cache *s; 205 + unsigned int delay; 206 + int i; 207 + 208 + if (IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST)) 209 + kunit_skip(test, "can't do kfree_rcu() when test is built-in"); 210 + 211 + INIT_WORK_ONSTACK(&cdw.work, cache_destroy_workfn); 212 + wq = alloc_workqueue("test_kfree_rcu_destroy_wq", 213 + WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 0); 214 + 215 + if (!wq) 216 + kunit_skip(test, "failed to alloc wq"); 217 + 218 + for (i = 0; i < KMEM_CACHE_DESTROY_NR; i++) { 219 + s = test_kmem_cache_create("TestSlub_kfree_rcu_wq_destroy", 220 + sizeof(struct test_kfree_rcu_struct), 221 + SLAB_NO_MERGE); 222 + 223 + if (!s) 224 + kunit_skip(test, "failed to create cache"); 225 + 226 + delay = get_random_u8(); 227 + p = kmem_cache_alloc(s, GFP_KERNEL); 228 + kfree_rcu(p, rcu); 229 + 230 + cdw.s = s; 231 + 232 + msleep(delay); 233 + queue_work(wq, &cdw.work); 234 + flush_work(&cdw.work); 235 + } 236 + 237 + destroy_workqueue(wq); 238 + KUNIT_EXPECT_EQ(test, 0, slab_errors); 239 + } 240 + 185 241 static void test_leak_destroy(struct kunit *test) 186 242 { 187 243 struct kmem_cache *s = test_kmem_cache_create("TestSlub_leak_destroy", ··· 312 254 KUNIT_CASE(test_clobber_redzone_free), 313 255 KUNIT_CASE(test_kmalloc_redzone_access), 314 256 KUNIT_CASE(test_kfree_rcu), 257 + KUNIT_CASE(test_kfree_rcu_wq_destroy), 315 258 KUNIT_CASE(test_leak_destroy), 316 259 KUNIT_CASE(test_krealloc_redzone_zeroing), 317 260 {}
+4
mm/Kconfig
··· 242 242 config SLUB 243 243 def_bool y 244 244 245 + config KVFREE_RCU_BATCHED 246 + def_bool y 247 + depends on !SLUB_TINY && !TINY_RCU 248 + 245 249 config SLUB_TINY 246 250 bool "Configure for minimal memory footprint" 247 251 depends on EXPERT
+7 -27
mm/slab.h
··· 457 457 return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT)); 458 458 } 459 459 460 - /* Legal flag mask for kmem_cache_create(), for various configurations */ 461 460 #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \ 462 461 SLAB_CACHE_DMA32 | SLAB_PANIC | \ 463 - SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS ) 462 + SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS | \ 463 + SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ 464 + SLAB_TEMPORARY | SLAB_ACCOUNT | \ 465 + SLAB_NO_USER_FLAGS | SLAB_KMALLOC | SLAB_NO_MERGE) 464 466 465 - #ifdef CONFIG_SLUB_DEBUG 466 467 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 467 468 SLAB_TRACE | SLAB_CONSISTENCY_CHECKS) 468 - #else 469 - #define SLAB_DEBUG_FLAGS (0) 470 - #endif 471 469 472 - #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ 473 - SLAB_TEMPORARY | SLAB_ACCOUNT | \ 474 - SLAB_NO_USER_FLAGS | SLAB_KMALLOC | SLAB_NO_MERGE) 475 - 476 - /* Common flags available with current configuration */ 477 - #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS) 478 - 479 - /* Common flags permitted for kmem_cache_create */ 480 - #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \ 481 - SLAB_RED_ZONE | \ 482 - SLAB_POISON | \ 483 - SLAB_STORE_USER | \ 484 - SLAB_TRACE | \ 485 - SLAB_CONSISTENCY_CHECKS | \ 486 - SLAB_NOLEAKTRACE | \ 487 - SLAB_RECLAIM_ACCOUNT | \ 488 - SLAB_TEMPORARY | \ 489 - SLAB_ACCOUNT | \ 490 - SLAB_KMALLOC | \ 491 - SLAB_NO_MERGE | \ 492 - SLAB_NO_USER_FLAGS) 470 + #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS) 493 471 494 472 bool __kmem_cache_empty(struct kmem_cache *); 495 473 int __kmem_cache_shutdown(struct kmem_cache *); ··· 581 603 void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 582 604 void **p, int objects, struct slabobj_ext *obj_exts); 583 605 #endif 606 + 607 + void kvfree_rcu_cb(struct rcu_head *head); 584 608 585 609 size_t __ksize(const void *objp); 586 610
+29 -15
mm/slab_common.c
··· 298 298 static_branch_enable(&slub_debug_enabled); 299 299 if (flags & SLAB_STORE_USER) 300 300 stack_depot_init(); 301 + #else 302 + flags &= ~SLAB_DEBUG_FLAGS; 301 303 #endif 302 304 303 305 mutex_lock(&slab_mutex); ··· 309 307 goto out_unlock; 310 308 } 311 309 312 - /* Refuse requests with allocator specific flags */ 313 310 if (flags & ~SLAB_FLAGS_PERMITTED) { 314 311 err = -EINVAL; 315 312 goto out_unlock; 316 313 } 317 - 318 - /* 319 - * Some allocators will constraint the set of valid flags to a subset 320 - * of all flags. We expect them to define CACHE_CREATE_MASK in this 321 - * case, and we'll just provide them with a sanitized version of the 322 - * passed flags. 323 - */ 324 - flags &= CACHE_CREATE_MASK; 325 314 326 315 /* Fail closed on bad usersize of useroffset values. */ 327 316 if (!IS_ENABLED(CONFIG_HARDENED_USERCOPY) || ··· 1277 1284 EXPORT_TRACEPOINT_SYMBOL(kfree); 1278 1285 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); 1279 1286 1287 + #ifndef CONFIG_KVFREE_RCU_BATCHED 1288 + 1289 + void kvfree_call_rcu(struct rcu_head *head, void *ptr) 1290 + { 1291 + if (head) { 1292 + kasan_record_aux_stack(ptr); 1293 + call_rcu(head, kvfree_rcu_cb); 1294 + return; 1295 + } 1296 + 1297 + // kvfree_rcu(one_arg) call. 1298 + might_sleep(); 1299 + synchronize_rcu(); 1300 + kvfree(ptr); 1301 + } 1302 + EXPORT_SYMBOL_GPL(kvfree_call_rcu); 1303 + 1304 + void __init kvfree_rcu_init(void) 1305 + { 1306 + } 1307 + 1308 + #else /* CONFIG_KVFREE_RCU_BATCHED */ 1309 + 1280 1310 /* 1281 1311 * This rcu parameter is runtime-read-only. It reflects 1282 1312 * a minimum allowed number of objects which can be cached ··· 1550 1534 rcu_lock_acquire(&rcu_callback_map); 1551 1535 trace_rcu_invoke_kvfree_callback("slab", head, offset); 1552 1536 1553 - if (!WARN_ON_ONCE(!__is_kvfree_rcu_offset(offset))) 1554 - kvfree(ptr); 1537 + kvfree(ptr); 1555 1538 1556 1539 rcu_lock_release(&rcu_callback_map); 1557 1540 cond_resched_tasks_rcu_qs(); ··· 1878 1863 return true; 1879 1864 } 1880 1865 1881 - #if !defined(CONFIG_TINY_RCU) 1882 - 1883 1866 static enum hrtimer_restart 1884 1867 schedule_page_work_fn(struct hrtimer *t) 1885 1868 { ··· 2086 2073 } 2087 2074 EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); 2088 2075 2089 - #endif /* #if !defined(CONFIG_TINY_RCU) */ 2090 - 2091 2076 static unsigned long 2092 2077 kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 2093 2078 { ··· 2179 2168 2180 2169 shrinker_register(kfree_rcu_shrinker); 2181 2170 } 2171 + 2172 + #endif /* CONFIG_KVFREE_RCU_BATCHED */ 2173 +
+287 -49
mm/slub.c
··· 19 19 #include <linux/bitops.h> 20 20 #include <linux/slab.h> 21 21 #include "slab.h" 22 + #include <linux/vmalloc.h> 22 23 #include <linux/proc_fs.h> 23 24 #include <linux/seq_file.h> 24 25 #include <linux/kasan.h> ··· 1018 1017 set_orig_size(s, (void *)object, s->object_size); 1019 1018 } 1020 1019 1021 - static void slab_bug(struct kmem_cache *s, char *fmt, ...) 1020 + static void __slab_bug(struct kmem_cache *s, const char *fmt, va_list argsp) 1022 1021 { 1023 1022 struct va_format vaf; 1024 1023 va_list args; 1025 1024 1026 - va_start(args, fmt); 1025 + va_copy(args, argsp); 1027 1026 vaf.fmt = fmt; 1028 1027 vaf.va = &args; 1029 1028 pr_err("=============================================================================\n"); 1030 - pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf); 1029 + pr_err("BUG %s (%s): %pV\n", s ? s->name : "<unknown>", print_tainted(), &vaf); 1031 1030 pr_err("-----------------------------------------------------------------------------\n\n"); 1032 1031 va_end(args); 1033 1032 } 1034 1033 1034 + static void slab_bug(struct kmem_cache *s, const char *fmt, ...) 1035 + { 1036 + va_list args; 1037 + 1038 + va_start(args, fmt); 1039 + __slab_bug(s, fmt, args); 1040 + va_end(args); 1041 + } 1042 + 1035 1043 __printf(2, 3) 1036 - static void slab_fix(struct kmem_cache *s, char *fmt, ...) 1044 + static void slab_fix(struct kmem_cache *s, const char *fmt, ...) 1037 1045 { 1038 1046 struct va_format vaf; 1039 1047 va_list args; ··· 1095 1085 /* Beginning of the filler is the free pointer */ 1096 1086 print_section(KERN_ERR, "Padding ", p + off, 1097 1087 size_from_object(s) - off); 1098 - 1099 - dump_stack(); 1100 1088 } 1101 1089 1102 1090 static void object_err(struct kmem_cache *s, struct slab *slab, 1103 - u8 *object, char *reason) 1091 + u8 *object, const char *reason) 1104 1092 { 1105 1093 if (slab_add_kunit_errors()) 1106 1094 return; 1107 1095 1108 - slab_bug(s, "%s", reason); 1096 + slab_bug(s, reason); 1109 1097 print_trailer(s, slab, object); 1110 1098 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1099 + 1100 + WARN_ON(1); 1111 1101 } 1112 1102 1113 1103 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, ··· 1124 1114 return false; 1125 1115 } 1126 1116 1117 + static void __slab_err(struct slab *slab) 1118 + { 1119 + if (slab_in_kunit_test()) 1120 + return; 1121 + 1122 + print_slab_info(slab); 1123 + add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1124 + 1125 + WARN_ON(1); 1126 + } 1127 + 1127 1128 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab, 1128 1129 const char *fmt, ...) 1129 1130 { 1130 1131 va_list args; 1131 - char buf[100]; 1132 1132 1133 1133 if (slab_add_kunit_errors()) 1134 1134 return; 1135 1135 1136 1136 va_start(args, fmt); 1137 - vsnprintf(buf, sizeof(buf), fmt, args); 1137 + __slab_bug(s, fmt, args); 1138 1138 va_end(args); 1139 - slab_bug(s, "%s", buf); 1140 - print_slab_info(slab); 1141 - dump_stack(); 1142 - add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1139 + 1140 + __slab_err(slab); 1143 1141 } 1144 1142 1145 1143 static void init_object(struct kmem_cache *s, void *object, u8 val) ··· 1184 1166 s->inuse - poison_size); 1185 1167 } 1186 1168 1187 - static void restore_bytes(struct kmem_cache *s, char *message, u8 data, 1169 + static void restore_bytes(struct kmem_cache *s, const char *message, u8 data, 1188 1170 void *from, void *to) 1189 1171 { 1190 1172 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data); ··· 1199 1181 1200 1182 static pad_check_attributes int 1201 1183 check_bytes_and_report(struct kmem_cache *s, struct slab *slab, 1202 - u8 *object, char *what, 1203 - u8 *start, unsigned int value, unsigned int bytes) 1184 + u8 *object, const char *what, u8 *start, unsigned int value, 1185 + unsigned int bytes, bool slab_obj_print) 1204 1186 { 1205 1187 u8 *fault; 1206 1188 u8 *end; ··· 1219 1201 if (slab_add_kunit_errors()) 1220 1202 goto skip_bug_print; 1221 1203 1222 - slab_bug(s, "%s overwritten", what); 1223 - pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", 1224 - fault, end - 1, fault - addr, 1225 - fault[0], value); 1204 + pr_err("[%s overwritten] 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", 1205 + what, fault, end - 1, fault - addr, fault[0], value); 1206 + 1207 + if (slab_obj_print) 1208 + object_err(s, slab, object, "Object corrupt"); 1226 1209 1227 1210 skip_bug_print: 1228 1211 restore_bytes(s, what, value, fault, end); ··· 1287 1268 return 1; 1288 1269 1289 1270 return check_bytes_and_report(s, slab, p, "Object padding", 1290 - p + off, POISON_INUSE, size_from_object(s) - off); 1271 + p + off, POISON_INUSE, size_from_object(s) - off, true); 1291 1272 } 1292 1273 1293 1274 /* Check the pad bytes at the end of a slab page */ ··· 1320 1301 while (end > fault && end[-1] == POISON_INUSE) 1321 1302 end--; 1322 1303 1323 - slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu", 1324 - fault, end - 1, fault - start); 1304 + slab_bug(s, "Padding overwritten. 0x%p-0x%p @offset=%tu", 1305 + fault, end - 1, fault - start); 1325 1306 print_section(KERN_ERR, "Padding ", pad, remainder); 1307 + __slab_err(slab); 1326 1308 1327 1309 restore_bytes(s, "slab padding", POISON_INUSE, fault, end); 1328 1310 } ··· 1338 1318 1339 1319 if (s->flags & SLAB_RED_ZONE) { 1340 1320 if (!check_bytes_and_report(s, slab, object, "Left Redzone", 1341 - object - s->red_left_pad, val, s->red_left_pad)) 1321 + object - s->red_left_pad, val, s->red_left_pad, ret)) 1342 1322 ret = 0; 1343 1323 1344 1324 if (!check_bytes_and_report(s, slab, object, "Right Redzone", 1345 - endobject, val, s->inuse - s->object_size)) 1325 + endobject, val, s->inuse - s->object_size, ret)) 1346 1326 ret = 0; 1347 1327 1348 1328 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { ··· 1351 1331 if (s->object_size > orig_size && 1352 1332 !check_bytes_and_report(s, slab, object, 1353 1333 "kmalloc Redzone", p + orig_size, 1354 - val, s->object_size - orig_size)) { 1334 + val, s->object_size - orig_size, ret)) { 1355 1335 ret = 0; 1356 1336 } 1357 1337 } ··· 1359 1339 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { 1360 1340 if (!check_bytes_and_report(s, slab, p, "Alignment padding", 1361 1341 endobject, POISON_INUSE, 1362 - s->inuse - s->object_size)) 1342 + s->inuse - s->object_size, ret)) 1363 1343 ret = 0; 1364 1344 } 1365 1345 } ··· 1375 1355 if (kasan_meta_size < s->object_size - 1 && 1376 1356 !check_bytes_and_report(s, slab, p, "Poison", 1377 1357 p + kasan_meta_size, POISON_FREE, 1378 - s->object_size - kasan_meta_size - 1)) 1358 + s->object_size - kasan_meta_size - 1, ret)) 1379 1359 ret = 0; 1380 1360 if (kasan_meta_size < s->object_size && 1381 1361 !check_bytes_and_report(s, slab, p, "End Poison", 1382 - p + s->object_size - 1, POISON_END, 1)) 1362 + p + s->object_size - 1, POISON_END, 1, ret)) 1383 1363 ret = 0; 1384 1364 } 1385 1365 /* ··· 1403 1383 */ 1404 1384 set_freepointer(s, p, NULL); 1405 1385 ret = 0; 1406 - } 1407 - 1408 - if (!ret && !slab_in_kunit_test()) { 1409 - print_trailer(s, slab, object); 1410 - add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1411 1386 } 1412 1387 1413 1388 return ret; ··· 1442 1427 * Determine if a certain object in a slab is on the freelist. Must hold the 1443 1428 * slab lock to guarantee that the chains are in a consistent state. 1444 1429 */ 1445 - static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search) 1430 + static bool on_freelist(struct kmem_cache *s, struct slab *slab, void *search) 1446 1431 { 1447 1432 int nr = 0; 1448 1433 void *fp; ··· 1452 1437 fp = slab->freelist; 1453 1438 while (fp && nr <= slab->objects) { 1454 1439 if (fp == search) 1455 - return 1; 1440 + return true; 1456 1441 if (!check_valid_pointer(s, slab, fp)) { 1457 1442 if (object) { 1458 1443 object_err(s, slab, object, 1459 1444 "Freechain corrupt"); 1460 1445 set_freepointer(s, object, NULL); 1446 + break; 1461 1447 } else { 1462 1448 slab_err(s, slab, "Freepointer corrupt"); 1463 1449 slab->freelist = NULL; 1464 1450 slab->inuse = slab->objects; 1465 1451 slab_fix(s, "Freelist cleared"); 1466 - return 0; 1452 + return false; 1467 1453 } 1468 - break; 1469 1454 } 1470 1455 object = fp; 1471 1456 fp = get_freepointer(s, object); 1472 1457 nr++; 1458 + } 1459 + 1460 + if (nr > slab->objects) { 1461 + slab_err(s, slab, "Freelist cycle detected"); 1462 + slab->freelist = NULL; 1463 + slab->inuse = slab->objects; 1464 + slab_fix(s, "Freelist cleared"); 1465 + return false; 1473 1466 } 1474 1467 1475 1468 max_objects = order_objects(slab_order(slab), s->size); ··· 1647 1624 slab_err(s, slab, "Attempt to free object(0x%p) outside of slab", 1648 1625 object); 1649 1626 } else if (!slab->slab_cache) { 1650 - pr_err("SLUB <none>: no slab for object 0x%p.\n", 1651 - object); 1652 - dump_stack(); 1653 - } else 1627 + slab_err(NULL, slab, "No slab cache for object 0x%p", 1628 + object); 1629 + } else { 1654 1630 object_err(s, slab, object, 1655 - "page slab pointer corrupt."); 1631 + "page slab pointer corrupt."); 1632 + } 1656 1633 return 0; 1657 1634 } 1658 1635 return 1; ··· 4264 4241 ptr = folio_address(folio); 4265 4242 lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, 4266 4243 PAGE_SIZE << order); 4244 + __folio_set_large_kmalloc(folio); 4267 4245 } 4268 4246 4269 4247 ptr = kasan_kmalloc_large(ptr, size, flags); ··· 4740 4716 { 4741 4717 unsigned int order = folio_order(folio); 4742 4718 4719 + if (WARN_ON_ONCE(!folio_test_large_kmalloc(folio))) { 4720 + dump_page(&folio->page, "Not a kmalloc allocation"); 4721 + return; 4722 + } 4723 + 4743 4724 if (WARN_ON_ONCE(order == 0)) 4744 4725 pr_warn_once("object pointer: 0x%p\n", object); 4745 4726 ··· 4754 4725 4755 4726 lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, 4756 4727 -(PAGE_SIZE << order)); 4728 + __folio_clear_large_kmalloc(folio); 4757 4729 folio_put(folio); 4730 + } 4731 + 4732 + /* 4733 + * Given an rcu_head embedded within an object obtained from kvmalloc at an 4734 + * offset < 4k, free the object in question. 4735 + */ 4736 + void kvfree_rcu_cb(struct rcu_head *head) 4737 + { 4738 + void *obj = head; 4739 + struct folio *folio; 4740 + struct slab *slab; 4741 + struct kmem_cache *s; 4742 + void *slab_addr; 4743 + 4744 + if (is_vmalloc_addr(obj)) { 4745 + obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); 4746 + vfree(obj); 4747 + return; 4748 + } 4749 + 4750 + folio = virt_to_folio(obj); 4751 + if (!folio_test_slab(folio)) { 4752 + /* 4753 + * rcu_head offset can be only less than page size so no need to 4754 + * consider folio order 4755 + */ 4756 + obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); 4757 + free_large_kmalloc(folio, obj); 4758 + return; 4759 + } 4760 + 4761 + slab = folio_slab(folio); 4762 + s = slab->slab_cache; 4763 + slab_addr = folio_address(folio); 4764 + 4765 + if (is_kfence_address(obj)) { 4766 + obj = kfence_object_start(obj); 4767 + } else { 4768 + unsigned int idx = __obj_to_index(s, slab_addr, obj); 4769 + 4770 + obj = slab_addr + s->size * idx; 4771 + obj = fixup_red_left(s, obj); 4772 + } 4773 + 4774 + slab_free(s, slab, obj, _RET_IP_); 4758 4775 } 4759 4776 4760 4777 /** ··· 4952 4877 return ret; 4953 4878 } 4954 4879 EXPORT_SYMBOL(krealloc_noprof); 4880 + 4881 + static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size) 4882 + { 4883 + /* 4884 + * We want to attempt a large physically contiguous block first because 4885 + * it is less likely to fragment multiple larger blocks and therefore 4886 + * contribute to a long term fragmentation less than vmalloc fallback. 4887 + * However make sure that larger requests are not too disruptive - no 4888 + * OOM killer and no allocation failure warnings as we have a fallback. 4889 + */ 4890 + if (size > PAGE_SIZE) { 4891 + flags |= __GFP_NOWARN; 4892 + 4893 + if (!(flags & __GFP_RETRY_MAYFAIL)) 4894 + flags |= __GFP_NORETRY; 4895 + 4896 + /* nofail semantic is implemented by the vmalloc fallback */ 4897 + flags &= ~__GFP_NOFAIL; 4898 + } 4899 + 4900 + return flags; 4901 + } 4902 + 4903 + /** 4904 + * __kvmalloc_node - attempt to allocate physically contiguous memory, but upon 4905 + * failure, fall back to non-contiguous (vmalloc) allocation. 4906 + * @size: size of the request. 4907 + * @b: which set of kmalloc buckets to allocate from. 4908 + * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. 4909 + * @node: numa node to allocate from 4910 + * 4911 + * Uses kmalloc to get the memory but if the allocation fails then falls back 4912 + * to the vmalloc allocator. Use kvfree for freeing the memory. 4913 + * 4914 + * GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier. 4915 + * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is 4916 + * preferable to the vmalloc fallback, due to visible performance drawbacks. 4917 + * 4918 + * Return: pointer to the allocated memory of %NULL in case of failure 4919 + */ 4920 + void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) 4921 + { 4922 + void *ret; 4923 + 4924 + /* 4925 + * It doesn't really make sense to fallback to vmalloc for sub page 4926 + * requests 4927 + */ 4928 + ret = __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), 4929 + kmalloc_gfp_adjust(flags, size), 4930 + node, _RET_IP_); 4931 + if (ret || size <= PAGE_SIZE) 4932 + return ret; 4933 + 4934 + /* non-sleeping allocations are not supported by vmalloc */ 4935 + if (!gfpflags_allow_blocking(flags)) 4936 + return NULL; 4937 + 4938 + /* Don't even allow crazy sizes */ 4939 + if (unlikely(size > INT_MAX)) { 4940 + WARN_ON_ONCE(!(flags & __GFP_NOWARN)); 4941 + return NULL; 4942 + } 4943 + 4944 + /* 4945 + * kvmalloc() can always use VM_ALLOW_HUGE_VMAP, 4946 + * since the callers already cannot assume anything 4947 + * about the resulting pointer, and cannot play 4948 + * protection games. 4949 + */ 4950 + return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, 4951 + flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, 4952 + node, __builtin_return_address(0)); 4953 + } 4954 + EXPORT_SYMBOL(__kvmalloc_node_noprof); 4955 + 4956 + /** 4957 + * kvfree() - Free memory. 4958 + * @addr: Pointer to allocated memory. 4959 + * 4960 + * kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc(). 4961 + * It is slightly more efficient to use kfree() or vfree() if you are certain 4962 + * that you know which one to use. 4963 + * 4964 + * Context: Either preemptible task context or not-NMI interrupt. 4965 + */ 4966 + void kvfree(const void *addr) 4967 + { 4968 + if (is_vmalloc_addr(addr)) 4969 + vfree(addr); 4970 + else 4971 + kfree(addr); 4972 + } 4973 + EXPORT_SYMBOL(kvfree); 4974 + 4975 + /** 4976 + * kvfree_sensitive - Free a data object containing sensitive information. 4977 + * @addr: address of the data object to be freed. 4978 + * @len: length of the data object. 4979 + * 4980 + * Use the special memzero_explicit() function to clear the content of a 4981 + * kvmalloc'ed object containing sensitive data to make sure that the 4982 + * compiler won't optimize out the data clearing. 4983 + */ 4984 + void kvfree_sensitive(const void *addr, size_t len) 4985 + { 4986 + if (likely(!ZERO_OR_NULL_PTR(addr))) { 4987 + memzero_explicit((void *)addr, len); 4988 + kvfree(addr); 4989 + } 4990 + } 4991 + EXPORT_SYMBOL(kvfree_sensitive); 4992 + 4993 + /** 4994 + * kvrealloc - reallocate memory; contents remain unchanged 4995 + * @p: object to reallocate memory for 4996 + * @size: the size to reallocate 4997 + * @flags: the flags for the page level allocator 4998 + * 4999 + * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0 5000 + * and @p is not a %NULL pointer, the object pointed to is freed. 5001 + * 5002 + * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 5003 + * initial memory allocation, every subsequent call to this API for the same 5004 + * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 5005 + * __GFP_ZERO is not fully honored by this API. 5006 + * 5007 + * In any case, the contents of the object pointed to are preserved up to the 5008 + * lesser of the new and old sizes. 5009 + * 5010 + * This function must not be called concurrently with itself or kvfree() for the 5011 + * same memory allocation. 5012 + * 5013 + * Return: pointer to the allocated memory or %NULL in case of error 5014 + */ 5015 + void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) 5016 + { 5017 + void *n; 5018 + 5019 + if (is_vmalloc_addr(p)) 5020 + return vrealloc_noprof(p, size, flags); 5021 + 5022 + n = krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size)); 5023 + if (!n) { 5024 + /* We failed to krealloc(), fall back to kvmalloc(). */ 5025 + n = kvmalloc_noprof(size, flags); 5026 + if (!n) 5027 + return NULL; 5028 + 5029 + if (p) { 5030 + /* We already know that `p` is not a vmalloc address. */ 5031 + kasan_disable_current(); 5032 + memcpy(n, kasan_reset_tag(p), ksize(p)); 5033 + kasan_enable_current(); 5034 + 5035 + kfree(p); 5036 + } 5037 + } 5038 + 5039 + return n; 5040 + } 5041 + EXPORT_SYMBOL(kvrealloc_noprof); 4955 5042 4956 5043 struct detached_freelist { 4957 5044 struct slab *slab; ··· 5807 5570 return !!oo_objects(s->oo); 5808 5571 } 5809 5572 5810 - static void list_slab_objects(struct kmem_cache *s, struct slab *slab, 5811 - const char *text) 5573 + static void list_slab_objects(struct kmem_cache *s, struct slab *slab) 5812 5574 { 5813 5575 #ifdef CONFIG_SLUB_DEBUG 5814 5576 void *addr = slab_address(slab); 5815 5577 void *p; 5816 5578 5817 - slab_err(s, slab, text, s->name); 5579 + if (!slab_add_kunit_errors()) 5580 + slab_bug(s, "Objects remaining on __kmem_cache_shutdown()"); 5818 5581 5819 5582 spin_lock(&object_map_lock); 5820 5583 __fill_map(object_map, s, slab); ··· 5829 5592 } 5830 5593 } 5831 5594 spin_unlock(&object_map_lock); 5595 + 5596 + __slab_err(slab); 5832 5597 #endif 5833 5598 } 5834 5599 ··· 5851 5612 remove_partial(n, slab); 5852 5613 list_add(&slab->slab_list, &discard); 5853 5614 } else { 5854 - list_slab_objects(s, slab, 5855 - "Objects remaining in %s on __kmem_cache_shutdown()"); 5615 + list_slab_objects(s, slab); 5856 5616 } 5857 5617 } 5858 5618 spin_unlock_irq(&n->list_lock);
-162
mm/util.c
··· 615 615 } 616 616 EXPORT_SYMBOL(vm_mmap); 617 617 618 - static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size) 619 - { 620 - /* 621 - * We want to attempt a large physically contiguous block first because 622 - * it is less likely to fragment multiple larger blocks and therefore 623 - * contribute to a long term fragmentation less than vmalloc fallback. 624 - * However make sure that larger requests are not too disruptive - no 625 - * OOM killer and no allocation failure warnings as we have a fallback. 626 - */ 627 - if (size > PAGE_SIZE) { 628 - flags |= __GFP_NOWARN; 629 - 630 - if (!(flags & __GFP_RETRY_MAYFAIL)) 631 - flags |= __GFP_NORETRY; 632 - 633 - /* nofail semantic is implemented by the vmalloc fallback */ 634 - flags &= ~__GFP_NOFAIL; 635 - } 636 - 637 - return flags; 638 - } 639 - 640 - /** 641 - * __kvmalloc_node - attempt to allocate physically contiguous memory, but upon 642 - * failure, fall back to non-contiguous (vmalloc) allocation. 643 - * @size: size of the request. 644 - * @b: which set of kmalloc buckets to allocate from. 645 - * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. 646 - * @node: numa node to allocate from 647 - * 648 - * Uses kmalloc to get the memory but if the allocation fails then falls back 649 - * to the vmalloc allocator. Use kvfree for freeing the memory. 650 - * 651 - * GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier. 652 - * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is 653 - * preferable to the vmalloc fallback, due to visible performance drawbacks. 654 - * 655 - * Return: pointer to the allocated memory of %NULL in case of failure 656 - */ 657 - void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) 658 - { 659 - void *ret; 660 - 661 - /* 662 - * It doesn't really make sense to fallback to vmalloc for sub page 663 - * requests 664 - */ 665 - ret = __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, b), 666 - kmalloc_gfp_adjust(flags, size), 667 - node); 668 - if (ret || size <= PAGE_SIZE) 669 - return ret; 670 - 671 - /* non-sleeping allocations are not supported by vmalloc */ 672 - if (!gfpflags_allow_blocking(flags)) 673 - return NULL; 674 - 675 - /* Don't even allow crazy sizes */ 676 - if (unlikely(size > INT_MAX)) { 677 - WARN_ON_ONCE(!(flags & __GFP_NOWARN)); 678 - return NULL; 679 - } 680 - 681 - /* 682 - * kvmalloc() can always use VM_ALLOW_HUGE_VMAP, 683 - * since the callers already cannot assume anything 684 - * about the resulting pointer, and cannot play 685 - * protection games. 686 - */ 687 - return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, 688 - flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, 689 - node, __builtin_return_address(0)); 690 - } 691 - EXPORT_SYMBOL(__kvmalloc_node_noprof); 692 - 693 - /** 694 - * kvfree() - Free memory. 695 - * @addr: Pointer to allocated memory. 696 - * 697 - * kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc(). 698 - * It is slightly more efficient to use kfree() or vfree() if you are certain 699 - * that you know which one to use. 700 - * 701 - * Context: Either preemptible task context or not-NMI interrupt. 702 - */ 703 - void kvfree(const void *addr) 704 - { 705 - if (is_vmalloc_addr(addr)) 706 - vfree(addr); 707 - else 708 - kfree(addr); 709 - } 710 - EXPORT_SYMBOL(kvfree); 711 - 712 - /** 713 - * kvfree_sensitive - Free a data object containing sensitive information. 714 - * @addr: address of the data object to be freed. 715 - * @len: length of the data object. 716 - * 717 - * Use the special memzero_explicit() function to clear the content of a 718 - * kvmalloc'ed object containing sensitive data to make sure that the 719 - * compiler won't optimize out the data clearing. 720 - */ 721 - void kvfree_sensitive(const void *addr, size_t len) 722 - { 723 - if (likely(!ZERO_OR_NULL_PTR(addr))) { 724 - memzero_explicit((void *)addr, len); 725 - kvfree(addr); 726 - } 727 - } 728 - EXPORT_SYMBOL(kvfree_sensitive); 729 - 730 - /** 731 - * kvrealloc - reallocate memory; contents remain unchanged 732 - * @p: object to reallocate memory for 733 - * @size: the size to reallocate 734 - * @flags: the flags for the page level allocator 735 - * 736 - * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0 737 - * and @p is not a %NULL pointer, the object pointed to is freed. 738 - * 739 - * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 740 - * initial memory allocation, every subsequent call to this API for the same 741 - * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 742 - * __GFP_ZERO is not fully honored by this API. 743 - * 744 - * In any case, the contents of the object pointed to are preserved up to the 745 - * lesser of the new and old sizes. 746 - * 747 - * This function must not be called concurrently with itself or kvfree() for the 748 - * same memory allocation. 749 - * 750 - * Return: pointer to the allocated memory or %NULL in case of error 751 - */ 752 - void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) 753 - { 754 - void *n; 755 - 756 - if (is_vmalloc_addr(p)) 757 - return vrealloc_noprof(p, size, flags); 758 - 759 - n = krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size)); 760 - if (!n) { 761 - /* We failed to krealloc(), fall back to kvmalloc(). */ 762 - n = kvmalloc_noprof(size, flags); 763 - if (!n) 764 - return NULL; 765 - 766 - if (p) { 767 - /* We already know that `p` is not a vmalloc address. */ 768 - kasan_disable_current(); 769 - memcpy(n, kasan_reset_tag(p), ksize(p)); 770 - kasan_enable_current(); 771 - 772 - kfree(p); 773 - } 774 - } 775 - 776 - return n; 777 - } 778 - EXPORT_SYMBOL(kvrealloc_noprof); 779 - 780 618 /** 781 619 * __vmalloc_array - allocate memory for a virtually contiguous array. 782 620 * @n: number of elements.