mempool: add mempool_{alloc,free}_bulk · tjh.dev/kernel@ac529d8

+141 -42

2 changed files

expand all

include

linux

mempool.h

mempool.c

include/linux/mempool.h

··· 66 66 extern void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask) __malloc; 67 67 #define mempool_alloc(...) \ 68 68 alloc_hooks(mempool_alloc_noprof(__VA_ARGS__)) 69 + int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem, 70 + unsigned int count, unsigned int allocated); 71 + #define mempool_alloc_bulk(...) \ 72 + alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__)) 69 73 70 74 extern void *mempool_alloc_preallocated(mempool_t *pool) __malloc; 71 75 extern void mempool_free(void *element, mempool_t *pool); 76 + unsigned int mempool_free_bulk(struct mempool *pool, void **elem, 77 + unsigned int count); 72 78 73 79 /* 74 80 * A mempool_alloc_t and mempool_free_t that get the memory from

+135 -42

mm/mempool.c

··· 21 21 #include "slab.h" 22 22 23 23 static DECLARE_FAULT_ATTR(fail_mempool_alloc); 24 + static DECLARE_FAULT_ATTR(fail_mempool_alloc_bulk); 24 25 25 26 static int __init mempool_faul_inject_init(void) 26 27 { 27 - return PTR_ERR_OR_ZERO(fault_create_debugfs_attr("fail_mempool_alloc", 28 + int error; 29 + 30 + error = PTR_ERR_OR_ZERO(fault_create_debugfs_attr("fail_mempool_alloc", 28 31 NULL, &fail_mempool_alloc)); 32 + if (error) 33 + return error; 34 + 35 + /* booting will fail on error return here, don't bother to cleanup */ 36 + return PTR_ERR_OR_ZERO( 37 + fault_create_debugfs_attr("fail_mempool_alloc_bulk", NULL, 38 + &fail_mempool_alloc_bulk)); 29 39 } 30 40 late_initcall(mempool_faul_inject_init); 31 41 ··· 390 380 } 391 381 EXPORT_SYMBOL(mempool_resize); 392 382 393 - static void *mempool_alloc_from_pool(struct mempool *pool, gfp_t gfp_mask) 383 + static unsigned int mempool_alloc_from_pool(struct mempool *pool, void **elems, 384 + unsigned int count, unsigned int allocated, 385 + gfp_t gfp_mask) 394 386 { 395 387 unsigned long flags; 396 - void *element; 388 + unsigned int i; 397 389 398 390 spin_lock_irqsave(&pool->lock, flags); 399 - if (unlikely(!pool->curr_nr)) 391 + if (unlikely(pool->curr_nr < count - allocated)) 400 392 goto fail; 401 - element = remove_element(pool); 393 + for (i = 0; i < count; i++) { 394 + if (!elems[i]) { 395 + elems[i] = remove_element(pool); 396 + allocated++; 397 + } 398 + } 402 399 spin_unlock_irqrestore(&pool->lock, flags); 403 400 404 401 /* Paired with rmb in mempool_free(), read comment there. */ ··· 415 398 * Update the allocation stack trace as this is more useful for 416 399 * debugging. 417 400 */ 418 - kmemleak_update_trace(element); 419 - return element; 401 + for (i = 0; i < count; i++) 402 + kmemleak_update_trace(elems[i]); 403 + return allocated; 420 404 421 405 fail: 422 406 if (gfp_mask & __GFP_DIRECT_RECLAIM) { ··· 439 421 spin_unlock_irqrestore(&pool->lock, flags); 440 422 } 441 423 442 - return NULL; 424 + return allocated; 443 425 } 444 426 445 427 /* ··· 454 436 *gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 455 437 return *gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO); 456 438 } 439 + 440 + /** 441 + * mempool_alloc_bulk - allocate multiple elements from a memory pool 442 + * @pool: pointer to the memory pool 443 + * @elems: partially or fully populated elements array 444 + * @count: number of entries in @elem that need to be allocated 445 + * @allocated: number of entries in @elem already allocated 446 + * 447 + * Allocate elements for each slot in @elem that is non-%NULL. This is done by 448 + * first calling into the alloc_fn supplied at pool initialization time, and 449 + * dipping into the reserved pool when alloc_fn fails to allocate an element. 450 + * 451 + * On return all @count elements in @elems will be populated. 452 + * 453 + * Return: Always 0. If it wasn't for %$#^$ alloc tags, it would return void. 454 + */ 455 + int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems, 456 + unsigned int count, unsigned int allocated) 457 + { 458 + gfp_t gfp_mask = GFP_KERNEL; 459 + gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask); 460 + unsigned int i = 0; 461 + 462 + VM_WARN_ON_ONCE(count > pool->min_nr); 463 + might_alloc(gfp_mask); 464 + 465 + /* 466 + * If an error is injected, fail all elements in a bulk allocation so 467 + * that we stress the multiple elements missing path. 468 + */ 469 + if (should_fail_ex(&fail_mempool_alloc_bulk, 1, FAULT_NOWARN)) { 470 + pr_info("forcing mempool usage for %pS\n", 471 + (void *)_RET_IP_); 472 + goto use_pool; 473 + } 474 + 475 + repeat_alloc: 476 + /* 477 + * Try to allocate the elements using the allocation callback first as 478 + * that might succeed even when the caller's bulk allocation did not. 479 + */ 480 + for (i = 0; i < count; i++) { 481 + if (elems[i]) 482 + continue; 483 + elems[i] = pool->alloc(gfp_temp, pool->pool_data); 484 + if (unlikely(!elems[i])) 485 + goto use_pool; 486 + allocated++; 487 + } 488 + 489 + return 0; 490 + 491 + use_pool: 492 + allocated = mempool_alloc_from_pool(pool, elems, count, allocated, 493 + gfp_temp); 494 + gfp_temp = gfp_mask; 495 + goto repeat_alloc; 496 + } 497 + EXPORT_SYMBOL_GPL(mempool_alloc_bulk_noprof); 457 498 458 499 /** 459 500 * mempool_alloc - allocate an element from a memory pool ··· 555 478 * sleep in mempool_alloc_from_pool. Retry the allocation 556 479 * with all flags set in that case. 557 480 */ 558 - element = mempool_alloc_from_pool(pool, gfp_temp); 559 - if (!element) { 481 + if (!mempool_alloc_from_pool(pool, &element, 1, 0, gfp_temp)) { 560 482 if (gfp_temp != gfp_mask) { 561 483 gfp_temp = gfp_mask; 562 484 goto repeat_alloc; ··· 584 508 */ 585 509 void *mempool_alloc_preallocated(mempool_t *pool) 586 510 { 587 - return mempool_alloc_from_pool(pool, GFP_NOWAIT); 511 + void *element = NULL; 512 + 513 + mempool_alloc_from_pool(pool, &element, 1, 0, GFP_NOWAIT); 514 + return element; 588 515 } 589 516 EXPORT_SYMBOL(mempool_alloc_preallocated); 590 517 591 518 /** 592 - * mempool_free - return an element to a mempool 593 - * @element: pointer to element 519 + * mempool_free_bulk - return elements to a mempool 594 520 * @pool: pointer to the memory pool 521 + * @elems: elements to return 522 + * @count: number of elements to return 595 523 * 596 - * Returns @element to @pool if it needs replenishing, else frees it using 597 - * the free_fn callback in @pool. 524 + * Returns a number of elements from the start of @elem to @pool if @pool needs 525 + * replenishing and sets their slots in @elem to NULL. Other elements are left 526 + * in @elem. 598 527 * 599 - * This function only sleeps if the free_fn callback sleeps. 528 + * Return: number of elements transferred to @pool. Elements are always 529 + * transferred from the beginning of @elem, so the return value can be used as 530 + * an offset into @elem for the freeing the remaining elements in the caller. 600 531 */ 601 - void mempool_free(void *element, mempool_t *pool) 532 + unsigned int mempool_free_bulk(struct mempool *pool, void **elems, 533 + unsigned int count) 602 534 { 603 535 unsigned long flags; 604 - 605 - if (unlikely(element == NULL)) 606 - return; 536 + unsigned int freed = 0; 537 + bool added = false; 607 538 608 539 /* 609 540 * Paired with the wmb in mempool_alloc(). The preceding read is ··· 644 561 * Waiters happen iff curr_nr is 0 and the above guarantee also 645 562 * ensures that there will be frees which return elements to the 646 563 * pool waking up the waiters. 647 - */ 648 - if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) { 649 - spin_lock_irqsave(&pool->lock, flags); 650 - if (likely(pool->curr_nr < pool->min_nr)) { 651 - add_element(pool, element); 652 - spin_unlock_irqrestore(&pool->lock, flags); 653 - if (wq_has_sleeper(&pool->wait)) 654 - wake_up(&pool->wait); 655 - return; 656 - } 657 - spin_unlock_irqrestore(&pool->lock, flags); 658 - } 659 - 660 - /* 661 - * Handle the min_nr = 0 edge case: 662 564 * 663 565 * For zero-minimum pools, curr_nr < min_nr (0 < 0) never succeeds, 664 566 * so waiters sleeping on pool->wait would never be woken by the ··· 651 583 * allocation of element when both min_nr and curr_nr are 0, and 652 584 * any active waiters are properly awakened. 653 585 */ 654 - if (unlikely(pool->min_nr == 0 && 586 + if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) { 587 + spin_lock_irqsave(&pool->lock, flags); 588 + while (pool->curr_nr < pool->min_nr && freed < count) { 589 + add_element(pool, elems[freed++]); 590 + added = true; 591 + } 592 + spin_unlock_irqrestore(&pool->lock, flags); 593 + } else if (unlikely(pool->min_nr == 0 && 655 594 READ_ONCE(pool->curr_nr) == 0)) { 595 + /* Handle the min_nr = 0 edge case: */ 656 596 spin_lock_irqsave(&pool->lock, flags); 657 597 if (likely(pool->curr_nr == 0)) { 658 - add_element(pool, element); 659 - spin_unlock_irqrestore(&pool->lock, flags); 660 - if (wq_has_sleeper(&pool->wait)) 661 - wake_up(&pool->wait); 662 - return; 598 + add_element(pool, elems[freed++]); 599 + added = true; 663 600 } 664 601 spin_unlock_irqrestore(&pool->lock, flags); 665 602 } 666 603 667 - pool->free(element, pool->pool_data); 604 + if (unlikely(added) && wq_has_sleeper(&pool->wait)) 605 + wake_up(&pool->wait); 606 + 607 + return freed; 608 + } 609 + EXPORT_SYMBOL_GPL(mempool_free_bulk); 610 + 611 + /** 612 + * mempool_free - return an element to the pool. 613 + * @element: element to return 614 + * @pool: pointer to the memory pool 615 + * 616 + * Returns @element to @pool if it needs replenishing, else frees it using 617 + * the free_fn callback in @pool. 618 + * 619 + * This function only sleeps if the free_fn callback sleeps. 620 + */ 621 + void mempool_free(void *element, struct mempool *pool) 622 + { 623 + if (likely(element) && !mempool_free_bulk(pool, &element, 1)) 624 + pool->free(element, pool->pool_data); 668 625 } 669 626 EXPORT_SYMBOL(mempool_free); 670 627

Configure Feed

Configure Feed