Merge branch 'slab/for-6.19/mempool_alloc_bulk' into slab/for-next

+4 -4

include/linux/fault-inject.h

··· 8 8 struct dentry; 9 9 struct kmem_cache; 10 10 11 + enum fault_flags { 12 + FAULT_NOWARN = 1 << 0, 13 + }; 14 + 11 15 #ifdef CONFIG_FAULT_INJECTION 12 16 13 17 #include <linux/atomic.h> ··· 38 34 unsigned long count; 39 35 struct ratelimit_state ratelimit_state; 40 36 struct dentry *dname; 41 - }; 42 - 43 - enum fault_flags { 44 - FAULT_NOWARN = 1 << 0, 45 37 }; 46 38 47 39 #define FAULT_ATTR_INITIALIZER { \

+25 -33

include/linux/mempool.h

··· 27 27 wait_queue_head_t wait; 28 28 } mempool_t; 29 29 30 - static inline bool mempool_initialized(mempool_t *pool) 30 + static inline bool mempool_initialized(struct mempool *pool) 31 31 { 32 32 return pool->elements != NULL; 33 33 } 34 34 35 - static inline bool mempool_is_saturated(mempool_t *pool) 35 + static inline bool mempool_is_saturated(struct mempool *pool) 36 36 { 37 37 return READ_ONCE(pool->curr_nr) >= pool->min_nr; 38 38 } 39 39 40 - void mempool_exit(mempool_t *pool); 41 - int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, 42 - mempool_free_t *free_fn, void *pool_data, 43 - gfp_t gfp_mask, int node_id); 44 - 45 - int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, 46 - mempool_free_t *free_fn, void *pool_data); 40 + void mempool_exit(struct mempool *pool); 41 + int mempool_init_node(struct mempool *pool, int min_nr, 42 + mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, 43 + void *pool_data, gfp_t gfp_mask, int node_id); 44 + int mempool_init_noprof(struct mempool *pool, int min_nr, 45 + mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, 46 + void *pool_data); 47 47 #define mempool_init(...) \ 48 48 alloc_hooks(mempool_init_noprof(__VA_ARGS__)) 49 49 50 - extern mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, 51 - mempool_free_t *free_fn, void *pool_data); 52 - 53 - extern mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn, 54 - mempool_free_t *free_fn, void *pool_data, 55 - gfp_t gfp_mask, int nid); 50 + struct mempool *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, 51 + mempool_free_t *free_fn, void *pool_data); 52 + struct mempool *mempool_create_node_noprof(int min_nr, 53 + mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, 54 + void *pool_data, gfp_t gfp_mask, int nid); 56 55 #define mempool_create_node(...) \ 57 56 alloc_hooks(mempool_create_node_noprof(__VA_ARGS__)) 58 57 ··· 59 60 mempool_create_node(_min_nr, _alloc_fn, _free_fn, _pool_data, \ 60 61 GFP_KERNEL, NUMA_NO_NODE) 61 62 62 - extern int mempool_resize(mempool_t *pool, int new_min_nr); 63 - extern void mempool_destroy(mempool_t *pool); 63 + int mempool_resize(struct mempool *pool, int new_min_nr); 64 + void mempool_destroy(struct mempool *pool); 64 65 65 - extern void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask) __malloc; 66 + void *mempool_alloc_noprof(struct mempool *pool, gfp_t gfp_mask) __malloc; 66 67 #define mempool_alloc(...) \ 67 68 alloc_hooks(mempool_alloc_noprof(__VA_ARGS__)) 69 + int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem, 70 + unsigned int count, unsigned int allocated); 71 + #define mempool_alloc_bulk(...) \ 72 + alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__)) 68 73 69 - extern void *mempool_alloc_preallocated(mempool_t *pool) __malloc; 70 - extern void mempool_free(void *element, mempool_t *pool); 74 + void *mempool_alloc_preallocated(struct mempool *pool) __malloc; 75 + void mempool_free(void *element, struct mempool *pool); 76 + unsigned int mempool_free_bulk(struct mempool *pool, void **elem, 77 + unsigned int count); 71 78 72 79 /* 73 80 * A mempool_alloc_t and mempool_free_t that get the memory from ··· 101 96 #define mempool_create_kmalloc_pool(_min_nr, _size) \ 102 97 mempool_create((_min_nr), mempool_kmalloc, mempool_kfree, \ 103 98 (void *)(unsigned long)(_size)) 104 - 105 - void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data); 106 - void mempool_kvfree(void *element, void *pool_data); 107 - 108 - static inline int mempool_init_kvmalloc_pool(mempool_t *pool, int min_nr, size_t size) 109 - { 110 - return mempool_init(pool, min_nr, mempool_kvmalloc, mempool_kvfree, (void *) size); 111 - } 112 - 113 - static inline mempool_t *mempool_create_kvmalloc_pool(int min_nr, size_t size) 114 - { 115 - return mempool_create(min_nr, mempool_kvmalloc, mempool_kvfree, (void *) size); 116 - } 117 99 118 100 /* 119 101 * A mempool_alloc_t and mempool_free_t for a simple page allocator that

+265 -162

mm/mempool.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 /* 3 - * linux/mm/mempool.c 4 - * 5 3 * memory buffer pool support. Such pools are mostly used 6 4 * for guaranteed, deadlock-free memory allocations during 7 5 * extreme VM load. ··· 7 9 * started by Ingo Molnar, Copyright (C) 2001 8 10 * debugging by David Rientjes, Copyright (C) 2015 9 11 */ 10 - 12 + #include <linux/fault-inject.h> 11 13 #include <linux/mm.h> 12 14 #include <linux/slab.h> 13 15 #include <linux/highmem.h> ··· 18 20 #include <linux/writeback.h> 19 21 #include "slab.h" 20 22 23 + static DECLARE_FAULT_ATTR(fail_mempool_alloc); 24 + static DECLARE_FAULT_ATTR(fail_mempool_alloc_bulk); 25 + 26 + static int __init mempool_faul_inject_init(void) 27 + { 28 + int error; 29 + 30 + error = PTR_ERR_OR_ZERO(fault_create_debugfs_attr("fail_mempool_alloc", 31 + NULL, &fail_mempool_alloc)); 32 + if (error) 33 + return error; 34 + 35 + /* booting will fail on error return here, don't bother to cleanup */ 36 + return PTR_ERR_OR_ZERO( 37 + fault_create_debugfs_attr("fail_mempool_alloc_bulk", NULL, 38 + &fail_mempool_alloc_bulk)); 39 + } 40 + late_initcall(mempool_faul_inject_init); 41 + 21 42 #ifdef CONFIG_SLUB_DEBUG_ON 22 - static void poison_error(mempool_t *pool, void *element, size_t size, 43 + static void poison_error(struct mempool *pool, void *element, size_t size, 23 44 size_t byte) 24 45 { 25 46 const int nr = pool->curr_nr; ··· 55 38 dump_stack(); 56 39 } 57 40 58 - static void __check_element(mempool_t *pool, void *element, size_t size) 41 + static void __check_element(struct mempool *pool, void *element, size_t size) 59 42 { 60 43 u8 *obj = element; 61 44 size_t i; ··· 71 54 memset(obj, POISON_INUSE, size); 72 55 } 73 56 74 - static void check_element(mempool_t *pool, void *element) 57 + static void check_element(struct mempool *pool, void *element) 75 58 { 76 59 /* Skip checking: KASAN might save its metadata in the element. */ 77 60 if (kasan_enabled()) ··· 100 83 obj[size - 1] = POISON_END; 101 84 } 102 85 103 - static void poison_element(mempool_t *pool, void *element) 86 + static void poison_element(struct mempool *pool, void *element) 104 87 { 105 88 /* Skip poisoning: KASAN might save its metadata in the element. */ 106 89 if (kasan_enabled()) ··· 121 104 } 122 105 } 123 106 #else /* CONFIG_SLUB_DEBUG_ON */ 124 - static inline void check_element(mempool_t *pool, void *element) 107 + static inline void check_element(struct mempool *pool, void *element) 125 108 { 126 109 } 127 - static inline void poison_element(mempool_t *pool, void *element) 110 + static inline void poison_element(struct mempool *pool, void *element) 128 111 { 129 112 } 130 113 #endif /* CONFIG_SLUB_DEBUG_ON */ 131 114 132 - static __always_inline bool kasan_poison_element(mempool_t *pool, void *element) 115 + static __always_inline bool kasan_poison_element(struct mempool *pool, 116 + void *element) 133 117 { 134 118 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) 135 119 return kasan_mempool_poison_object(element); ··· 140 122 return true; 141 123 } 142 124 143 - static void kasan_unpoison_element(mempool_t *pool, void *element) 125 + static void kasan_unpoison_element(struct mempool *pool, void *element) 144 126 { 145 127 if (pool->alloc == mempool_kmalloc) 146 128 kasan_mempool_unpoison_object(element, (size_t)pool->pool_data); ··· 152 134 (unsigned long)pool->pool_data); 153 135 } 154 136 155 - static __always_inline void add_element(mempool_t *pool, void *element) 137 + static __always_inline void add_element(struct mempool *pool, void *element) 156 138 { 157 139 BUG_ON(pool->min_nr != 0 && pool->curr_nr >= pool->min_nr); 158 140 poison_element(pool, element); ··· 160 142 pool->elements[pool->curr_nr++] = element; 161 143 } 162 144 163 - static void *remove_element(mempool_t *pool) 145 + static void *remove_element(struct mempool *pool) 164 146 { 165 147 void *element = pool->elements[--pool->curr_nr]; 166 148 ··· 181 163 * May be called on a zeroed but uninitialized mempool (i.e. allocated with 182 164 * kzalloc()). 183 165 */ 184 - void mempool_exit(mempool_t *pool) 166 + void mempool_exit(struct mempool *pool) 185 167 { 186 168 while (pool->curr_nr) { 187 169 void *element = remove_element(pool); ··· 200 182 * Free all reserved elements in @pool and @pool itself. This function 201 183 * only sleeps if the free_fn() function sleeps. 202 184 */ 203 - void mempool_destroy(mempool_t *pool) 185 + void mempool_destroy(struct mempool *pool) 204 186 { 205 187 if (unlikely(!pool)) 206 188 return; ··· 210 192 } 211 193 EXPORT_SYMBOL(mempool_destroy); 212 194 213 - int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, 214 - mempool_free_t *free_fn, void *pool_data, 215 - gfp_t gfp_mask, int node_id) 195 + int mempool_init_node(struct mempool *pool, int min_nr, 196 + mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, 197 + void *pool_data, gfp_t gfp_mask, int node_id) 216 198 { 217 199 spin_lock_init(&pool->lock); 218 200 pool->min_nr = min_nr; ··· 262 244 * 263 245 * Return: %0 on success, negative error code otherwise. 264 246 */ 265 - int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, 266 - mempool_free_t *free_fn, void *pool_data) 247 + int mempool_init_noprof(struct mempool *pool, int min_nr, 248 + mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, 249 + void *pool_data) 267 250 { 268 251 return mempool_init_node(pool, min_nr, alloc_fn, free_fn, 269 252 pool_data, GFP_KERNEL, NUMA_NO_NODE); ··· 290 271 * 291 272 * Return: pointer to the created memory pool object or %NULL on error. 292 273 */ 293 - mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn, 294 - mempool_free_t *free_fn, void *pool_data, 295 - gfp_t gfp_mask, int node_id) 274 + struct mempool *mempool_create_node_noprof(int min_nr, 275 + mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, 276 + void *pool_data, gfp_t gfp_mask, int node_id) 296 277 { 297 - mempool_t *pool; 278 + struct mempool *pool; 298 279 299 280 pool = kmalloc_node_noprof(sizeof(*pool), gfp_mask | __GFP_ZERO, node_id); 300 281 if (!pool) ··· 328 309 * 329 310 * Return: %0 on success, negative error code otherwise. 330 311 */ 331 - int mempool_resize(mempool_t *pool, int new_min_nr) 312 + int mempool_resize(struct mempool *pool, int new_min_nr) 332 313 { 333 314 void *element; 334 315 void **new_elements; ··· 390 371 } 391 372 EXPORT_SYMBOL(mempool_resize); 392 373 393 - /** 394 - * mempool_alloc - allocate an element from a specific memory pool 395 - * @pool: pointer to the memory pool which was allocated via 396 - * mempool_create(). 397 - * @gfp_mask: the usual allocation bitmask. 398 - * 399 - * this function only sleeps if the alloc_fn() function sleeps or 400 - * returns NULL. Note that due to preallocation, this function 401 - * *never* fails when called from process contexts. (it might 402 - * fail if called from an IRQ context.) 403 - * Note: using __GFP_ZERO is not supported. 404 - * 405 - * Return: pointer to the allocated element or %NULL on error. 406 - */ 407 - void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask) 374 + static unsigned int mempool_alloc_from_pool(struct mempool *pool, void **elems, 375 + unsigned int count, unsigned int allocated, 376 + gfp_t gfp_mask) 408 377 { 409 - void *element; 410 378 unsigned long flags; 411 - wait_queue_entry_t wait; 412 - gfp_t gfp_temp; 379 + unsigned int i; 380 + 381 + spin_lock_irqsave(&pool->lock, flags); 382 + if (unlikely(pool->curr_nr < count - allocated)) 383 + goto fail; 384 + for (i = 0; i < count; i++) { 385 + if (!elems[i]) { 386 + elems[i] = remove_element(pool); 387 + allocated++; 388 + } 389 + } 390 + spin_unlock_irqrestore(&pool->lock, flags); 391 + 392 + /* Paired with rmb in mempool_free(), read comment there. */ 393 + smp_wmb(); 394 + 395 + /* 396 + * Update the allocation stack trace as this is more useful for 397 + * debugging. 398 + */ 399 + for (i = 0; i < count; i++) 400 + kmemleak_update_trace(elems[i]); 401 + return allocated; 402 + 403 + fail: 404 + if (gfp_mask & __GFP_DIRECT_RECLAIM) { 405 + DEFINE_WAIT(wait); 406 + 407 + prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 408 + spin_unlock_irqrestore(&pool->lock, flags); 409 + 410 + /* 411 + * Wait for someone else to return an element to @pool, but wake 412 + * up occasionally as memory pressure might have reduced even 413 + * and the normal allocation in alloc_fn could succeed even if 414 + * no element was returned. 415 + */ 416 + io_schedule_timeout(5 * HZ); 417 + finish_wait(&pool->wait, &wait); 418 + } else { 419 + /* We must not sleep if __GFP_DIRECT_RECLAIM is not set. */ 420 + spin_unlock_irqrestore(&pool->lock, flags); 421 + } 422 + 423 + return allocated; 424 + } 425 + 426 + /* 427 + * Adjust the gfp flags for mempool allocations, as we never want to dip into 428 + * the global emergency reserves or retry in the page allocator. 429 + * 430 + * The first pass also doesn't want to go reclaim, but the next passes do, so 431 + * return a separate subset for that first iteration. 432 + */ 433 + static inline gfp_t mempool_adjust_gfp(gfp_t *gfp_mask) 434 + { 435 + *gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 436 + return *gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO); 437 + } 438 + 439 + /** 440 + * mempool_alloc_bulk - allocate multiple elements from a memory pool 441 + * @pool: pointer to the memory pool 442 + * @elems: partially or fully populated elements array 443 + * @count: number of entries in @elem that need to be allocated 444 + * @allocated: number of entries in @elem already allocated 445 + * 446 + * Allocate elements for each slot in @elem that is non-%NULL. This is done by 447 + * first calling into the alloc_fn supplied at pool initialization time, and 448 + * dipping into the reserved pool when alloc_fn fails to allocate an element. 449 + * 450 + * On return all @count elements in @elems will be populated. 451 + * 452 + * Return: Always 0. If it wasn't for %$#^$ alloc tags, it would return void. 453 + */ 454 + int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems, 455 + unsigned int count, unsigned int allocated) 456 + { 457 + gfp_t gfp_mask = GFP_KERNEL; 458 + gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask); 459 + unsigned int i = 0; 460 + 461 + VM_WARN_ON_ONCE(count > pool->min_nr); 462 + might_alloc(gfp_mask); 463 + 464 + /* 465 + * If an error is injected, fail all elements in a bulk allocation so 466 + * that we stress the multiple elements missing path. 467 + */ 468 + if (should_fail_ex(&fail_mempool_alloc_bulk, 1, FAULT_NOWARN)) { 469 + pr_info("forcing mempool usage for %pS\n", 470 + (void *)_RET_IP_); 471 + goto use_pool; 472 + } 473 + 474 + repeat_alloc: 475 + /* 476 + * Try to allocate the elements using the allocation callback first as 477 + * that might succeed even when the caller's bulk allocation did not. 478 + */ 479 + for (i = 0; i < count; i++) { 480 + if (elems[i]) 481 + continue; 482 + elems[i] = pool->alloc(gfp_temp, pool->pool_data); 483 + if (unlikely(!elems[i])) 484 + goto use_pool; 485 + allocated++; 486 + } 487 + 488 + return 0; 489 + 490 + use_pool: 491 + allocated = mempool_alloc_from_pool(pool, elems, count, allocated, 492 + gfp_temp); 493 + gfp_temp = gfp_mask; 494 + goto repeat_alloc; 495 + } 496 + EXPORT_SYMBOL_GPL(mempool_alloc_bulk_noprof); 497 + 498 + /** 499 + * mempool_alloc - allocate an element from a memory pool 500 + * @pool: pointer to the memory pool 501 + * @gfp_mask: GFP_* flags. %__GFP_ZERO is not supported. 502 + * 503 + * Allocate an element from @pool. This is done by first calling into the 504 + * alloc_fn supplied at pool initialization time, and dipping into the reserved 505 + * pool when alloc_fn fails to allocate an element. 506 + * 507 + * This function only sleeps if the alloc_fn callback sleeps, or when waiting 508 + * for elements to become available in the pool. 509 + * 510 + * Return: pointer to the allocated element or %NULL when failing to allocate 511 + * an element. Allocation failure can only happen when @gfp_mask does not 512 + * include %__GFP_DIRECT_RECLAIM. 513 + */ 514 + void *mempool_alloc_noprof(struct mempool *pool, gfp_t gfp_mask) 515 + { 516 + gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask); 517 + void *element; 413 518 414 519 VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO); 415 520 might_alloc(gfp_mask); 416 521 417 - gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */ 418 - gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */ 419 - gfp_mask |= __GFP_NOWARN; /* failures are OK */ 420 - 421 - gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO); 422 - 423 522 repeat_alloc: 523 + if (should_fail_ex(&fail_mempool_alloc, 1, FAULT_NOWARN)) { 524 + pr_info("forcing mempool usage for %pS\n", 525 + (void *)_RET_IP_); 526 + element = NULL; 527 + } else { 528 + element = pool->alloc(gfp_temp, pool->pool_data); 529 + } 424 530 425 - element = pool->alloc(gfp_temp, pool->pool_data); 426 - if (likely(element != NULL)) 427 - return element; 428 - 429 - spin_lock_irqsave(&pool->lock, flags); 430 - if (likely(pool->curr_nr)) { 431 - element = remove_element(pool); 432 - spin_unlock_irqrestore(&pool->lock, flags); 433 - /* paired with rmb in mempool_free(), read comment there */ 434 - smp_wmb(); 531 + if (unlikely(!element)) { 435 532 /* 436 - * Update the allocation stack trace as this is more useful 437 - * for debugging. 533 + * Try to allocate an element from the pool. 534 + * 535 + * The first pass won't have __GFP_DIRECT_RECLAIM and won't 536 + * sleep in mempool_alloc_from_pool. Retry the allocation 537 + * with all flags set in that case. 438 538 */ 439 - kmemleak_update_trace(element); 440 - return element; 539 + if (!mempool_alloc_from_pool(pool, &element, 1, 0, gfp_temp)) { 540 + if (gfp_temp != gfp_mask) { 541 + gfp_temp = gfp_mask; 542 + goto repeat_alloc; 543 + } 544 + if (gfp_mask & __GFP_DIRECT_RECLAIM) { 545 + goto repeat_alloc; 546 + } 547 + } 441 548 } 442 549 443 - /* 444 - * We use gfp mask w/o direct reclaim or IO for the first round. If 445 - * alloc failed with that and @pool was empty, retry immediately. 446 - */ 447 - if (gfp_temp != gfp_mask) { 448 - spin_unlock_irqrestore(&pool->lock, flags); 449 - gfp_temp = gfp_mask; 450 - goto repeat_alloc; 451 - } 452 - 453 - /* We must not sleep if !__GFP_DIRECT_RECLAIM */ 454 - if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { 455 - spin_unlock_irqrestore(&pool->lock, flags); 456 - return NULL; 457 - } 458 - 459 - /* Let's wait for someone else to return an element to @pool */ 460 - init_wait(&wait); 461 - prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 462 - 463 - spin_unlock_irqrestore(&pool->lock, flags); 464 - 465 - /* 466 - * FIXME: this should be io_schedule(). The timeout is there as a 467 - * workaround for some DM problems in 2.6.18. 468 - */ 469 - io_schedule_timeout(5*HZ); 470 - 471 - finish_wait(&pool->wait, &wait); 472 - goto repeat_alloc; 550 + return element; 473 551 } 474 552 EXPORT_SYMBOL(mempool_alloc_noprof); 475 553 476 554 /** 477 555 * mempool_alloc_preallocated - allocate an element from preallocated elements 478 - * belonging to a specific memory pool 479 - * @pool: pointer to the memory pool which was allocated via 480 - * mempool_create(). 556 + * belonging to a memory pool 557 + * @pool: pointer to the memory pool 481 558 * 482 - * This function is similar to mempool_alloc, but it only attempts allocating 483 - * an element from the preallocated elements. It does not sleep and immediately 484 - * returns if no preallocated elements are available. 559 + * This function is similar to mempool_alloc(), but it only attempts allocating 560 + * an element from the preallocated elements. It only takes a single spinlock_t 561 + * and immediately returns if no preallocated elements are available. 485 562 * 486 563 * Return: pointer to the allocated element or %NULL if no elements are 487 564 * available. 488 565 */ 489 - void *mempool_alloc_preallocated(mempool_t *pool) 566 + void *mempool_alloc_preallocated(struct mempool *pool) 490 567 { 491 - void *element; 492 - unsigned long flags; 568 + void *element = NULL; 493 569 494 - spin_lock_irqsave(&pool->lock, flags); 495 - if (likely(pool->curr_nr)) { 496 - element = remove_element(pool); 497 - spin_unlock_irqrestore(&pool->lock, flags); 498 - /* paired with rmb in mempool_free(), read comment there */ 499 - smp_wmb(); 500 - /* 501 - * Update the allocation stack trace as this is more useful 502 - * for debugging. 503 - */ 504 - kmemleak_update_trace(element); 505 - return element; 506 - } 507 - spin_unlock_irqrestore(&pool->lock, flags); 508 - 509 - return NULL; 570 + mempool_alloc_from_pool(pool, &element, 1, 0, GFP_NOWAIT); 571 + return element; 510 572 } 511 573 EXPORT_SYMBOL(mempool_alloc_preallocated); 512 574 513 575 /** 514 - * mempool_free - return an element to the pool. 515 - * @element: pool element pointer. 516 - * @pool: pointer to the memory pool which was allocated via 517 - * mempool_create(). 576 + * mempool_free_bulk - return elements to a mempool 577 + * @pool: pointer to the memory pool 578 + * @elems: elements to return 579 + * @count: number of elements to return 518 580 * 519 - * this function only sleeps if the free_fn() function sleeps. 581 + * Returns a number of elements from the start of @elem to @pool if @pool needs 582 + * replenishing and sets their slots in @elem to NULL. Other elements are left 583 + * in @elem. 584 + * 585 + * Return: number of elements transferred to @pool. Elements are always 586 + * transferred from the beginning of @elem, so the return value can be used as 587 + * an offset into @elem for the freeing the remaining elements in the caller. 520 588 */ 521 - void mempool_free(void *element, mempool_t *pool) 589 + unsigned int mempool_free_bulk(struct mempool *pool, void **elems, 590 + unsigned int count) 522 591 { 523 592 unsigned long flags; 524 - 525 - if (unlikely(element == NULL)) 526 - return; 593 + unsigned int freed = 0; 594 + bool added = false; 527 595 528 596 /* 529 597 * Paired with the wmb in mempool_alloc(). The preceding read is ··· 644 538 * Waiters happen iff curr_nr is 0 and the above guarantee also 645 539 * ensures that there will be frees which return elements to the 646 540 * pool waking up the waiters. 647 - */ 648 - if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) { 649 - spin_lock_irqsave(&pool->lock, flags); 650 - if (likely(pool->curr_nr < pool->min_nr)) { 651 - add_element(pool, element); 652 - spin_unlock_irqrestore(&pool->lock, flags); 653 - if (wq_has_sleeper(&pool->wait)) 654 - wake_up(&pool->wait); 655 - return; 656 - } 657 - spin_unlock_irqrestore(&pool->lock, flags); 658 - } 659 - 660 - /* 661 - * Handle the min_nr = 0 edge case: 662 541 * 663 542 * For zero-minimum pools, curr_nr < min_nr (0 < 0) never succeeds, 664 543 * so waiters sleeping on pool->wait would never be woken by the ··· 651 560 * allocation of element when both min_nr and curr_nr are 0, and 652 561 * any active waiters are properly awakened. 653 562 */ 654 - if (unlikely(pool->min_nr == 0 && 563 + if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) { 564 + spin_lock_irqsave(&pool->lock, flags); 565 + while (pool->curr_nr < pool->min_nr && freed < count) { 566 + add_element(pool, elems[freed++]); 567 + added = true; 568 + } 569 + spin_unlock_irqrestore(&pool->lock, flags); 570 + } else if (unlikely(pool->min_nr == 0 && 655 571 READ_ONCE(pool->curr_nr) == 0)) { 572 + /* Handle the min_nr = 0 edge case: */ 656 573 spin_lock_irqsave(&pool->lock, flags); 657 574 if (likely(pool->curr_nr == 0)) { 658 - add_element(pool, element); 659 - spin_unlock_irqrestore(&pool->lock, flags); 660 - if (wq_has_sleeper(&pool->wait)) 661 - wake_up(&pool->wait); 662 - return; 575 + add_element(pool, elems[freed++]); 576 + added = true; 663 577 } 664 578 spin_unlock_irqrestore(&pool->lock, flags); 665 579 } 666 580 667 - pool->free(element, pool->pool_data); 581 + if (unlikely(added) && wq_has_sleeper(&pool->wait)) 582 + wake_up(&pool->wait); 583 + 584 + return freed; 585 + } 586 + EXPORT_SYMBOL_GPL(mempool_free_bulk); 587 + 588 + /** 589 + * mempool_free - return an element to the pool. 590 + * @element: element to return 591 + * @pool: pointer to the memory pool 592 + * 593 + * Returns @element to @pool if it needs replenishing, else frees it using 594 + * the free_fn callback in @pool. 595 + * 596 + * This function only sleeps if the free_fn callback sleeps. 597 + */ 598 + void mempool_free(void *element, struct mempool *pool) 599 + { 600 + if (likely(element) && !mempool_free_bulk(pool, &element, 1)) 601 + pool->free(element, pool->pool_data); 668 602 } 669 603 EXPORT_SYMBOL(mempool_free); 670 604 ··· 727 611 kfree(element); 728 612 } 729 613 EXPORT_SYMBOL(mempool_kfree); 730 - 731 - void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data) 732 - { 733 - size_t size = (size_t)pool_data; 734 - return kvmalloc(size, gfp_mask); 735 - } 736 - EXPORT_SYMBOL(mempool_kvmalloc); 737 - 738 - void mempool_kvfree(void *element, void *pool_data) 739 - { 740 - kvfree(element); 741 - } 742 - EXPORT_SYMBOL(mempool_kvfree); 743 614 744 615 /* 745 616 * A simple mempool-backed page allocator that allocates pages

+10 -5

mm/page_alloc.c

··· 4982 4982 * @nr_pages: The number of pages desired in the array 4983 4983 * @page_array: Array to store the pages 4984 4984 * 4985 - * This is a batched version of the page allocator that attempts to 4986 - * allocate nr_pages quickly. Pages are added to the page_array. 4985 + * This is a batched version of the page allocator that attempts to allocate 4986 + * @nr_pages quickly. Pages are added to @page_array. 4987 4987 * 4988 - * Note that only NULL elements are populated with pages and nr_pages 4989 - * is the maximum number of pages that will be stored in the array. 4988 + * Note that only the elements in @page_array that were cleared to %NULL on 4989 + * entry are populated with newly allocated pages. @nr_pages is the maximum 4990 + * number of pages that will be stored in the array. 4990 4991 * 4991 - * Returns the number of pages in the array. 4992 + * Returns the number of pages in @page_array, including ones already 4993 + * allocated on entry. This can be less than the number requested in @nr_pages, 4994 + * but all empty slots are filled from the beginning. I.e., if all slots in 4995 + * @page_array were set to %NULL on entry, the slots from 0 to the return value 4996 + * - 1 will be filled. 4992 4997 */ 4993 4998 unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, 4994 4999 nodemask_t *nodemask, int nr_pages,

Configure Feed

Configure Feed