Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mempool: factor out a mempool_alloc_from_pool helper
Add a helper for the mempool_alloc slowpath to better separate it from the
fast path, and also use it to implement mempool_alloc_preallocated which
shares the same logic.
[hughd@google.com: fix lack of retrying with __GFP_DIRECT_RECLAIM]
[vbabka@suse.cz: really use limited flags for first mempool attempt]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Link: https://patch.msgid.link/20251113084022.1255121-7-hch@lst.de
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
authored by
Christoph Hellwig
and committed by
Vlastimil Babka
1742d97d
3d249240
+62
-64
+62
-64
mm/mempool.c
+62
-64
mm/mempool.c
···
380
380
}
381
381
EXPORT_SYMBOL(mempool_resize);
382
382
383
+
static void *mempool_alloc_from_pool(struct mempool *pool, gfp_t gfp_mask)
384
+
{
385
+
unsigned long flags;
386
+
void *element;
387
+
388
+
spin_lock_irqsave(&pool->lock, flags);
389
+
if (unlikely(!pool->curr_nr))
390
+
goto fail;
391
+
element = remove_element(pool);
392
+
spin_unlock_irqrestore(&pool->lock, flags);
393
+
394
+
/* Paired with rmb in mempool_free(), read comment there. */
395
+
smp_wmb();
396
+
397
+
/*
398
+
* Update the allocation stack trace as this is more useful for
399
+
* debugging.
400
+
*/
401
+
kmemleak_update_trace(element);
402
+
return element;
403
+
404
+
fail:
405
+
if (gfp_mask & __GFP_DIRECT_RECLAIM) {
406
+
DEFINE_WAIT(wait);
407
+
408
+
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
409
+
spin_unlock_irqrestore(&pool->lock, flags);
410
+
411
+
/*
412
+
* Wait for someone else to return an element to @pool.
413
+
*
414
+
* FIXME: this should be io_schedule(). The timeout is there as
415
+
* a workaround for some DM problems in 2.6.18.
416
+
*/
417
+
io_schedule_timeout(5 * HZ);
418
+
finish_wait(&pool->wait, &wait);
419
+
} else {
420
+
/* We must not sleep if __GFP_DIRECT_RECLAIM is not set. */
421
+
spin_unlock_irqrestore(&pool->lock, flags);
422
+
}
423
+
424
+
return NULL;
425
+
}
426
+
383
427
/*
384
428
* Adjust the gfp flags for mempool allocations, as we never want to dip into
385
429
* the global emergency reserves or retry in the page allocator.
···
457
413
{
458
414
gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask);
459
415
void *element;
460
-
unsigned long flags;
461
-
wait_queue_entry_t wait;
462
416
463
417
VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
464
418
might_alloc(gfp_mask);
···
470
428
element = pool->alloc(gfp_temp, pool->pool_data);
471
429
}
472
430
473
-
if (likely(element))
474
-
return element;
475
-
476
-
spin_lock_irqsave(&pool->lock, flags);
477
-
if (likely(pool->curr_nr)) {
478
-
element = remove_element(pool);
479
-
spin_unlock_irqrestore(&pool->lock, flags);
480
-
/* paired with rmb in mempool_free(), read comment there */
481
-
smp_wmb();
431
+
if (unlikely(!element)) {
482
432
/*
483
-
* Update the allocation stack trace as this is more useful
484
-
* for debugging.
433
+
* Try to allocate an element from the pool.
434
+
*
435
+
* The first pass won't have __GFP_DIRECT_RECLAIM and won't
436
+
* sleep in mempool_alloc_from_pool. Retry the allocation
437
+
* with all flags set in that case.
485
438
*/
486
-
kmemleak_update_trace(element);
487
-
return element;
439
+
element = mempool_alloc_from_pool(pool, gfp_temp);
440
+
if (!element) {
441
+
if (gfp_temp != gfp_mask) {
442
+
gfp_temp = gfp_mask;
443
+
goto repeat_alloc;
444
+
}
445
+
if (gfp_mask & __GFP_DIRECT_RECLAIM) {
446
+
goto repeat_alloc;
447
+
}
448
+
}
488
449
}
489
450
490
-
/*
491
-
* We use gfp mask w/o direct reclaim or IO for the first round. If
492
-
* alloc failed with that and @pool was empty, retry immediately.
493
-
*/
494
-
if (gfp_temp != gfp_mask) {
495
-
spin_unlock_irqrestore(&pool->lock, flags);
496
-
gfp_temp = gfp_mask;
497
-
goto repeat_alloc;
498
-
}
499
-
500
-
/* We must not sleep if !__GFP_DIRECT_RECLAIM */
501
-
if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
502
-
spin_unlock_irqrestore(&pool->lock, flags);
503
-
return NULL;
504
-
}
505
-
506
-
/* Let's wait for someone else to return an element to @pool */
507
-
init_wait(&wait);
508
-
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
509
-
510
-
spin_unlock_irqrestore(&pool->lock, flags);
511
-
512
-
/*
513
-
* FIXME: this should be io_schedule(). The timeout is there as a
514
-
* workaround for some DM problems in 2.6.18.
515
-
*/
516
-
io_schedule_timeout(5*HZ);
517
-
518
-
finish_wait(&pool->wait, &wait);
519
-
goto repeat_alloc;
451
+
return element;
520
452
}
521
453
EXPORT_SYMBOL(mempool_alloc_noprof);
522
454
···
508
492
*/
509
493
void *mempool_alloc_preallocated(mempool_t *pool)
510
494
{
511
-
void *element;
512
-
unsigned long flags;
513
-
514
-
spin_lock_irqsave(&pool->lock, flags);
515
-
if (likely(pool->curr_nr)) {
516
-
element = remove_element(pool);
517
-
spin_unlock_irqrestore(&pool->lock, flags);
518
-
/* paired with rmb in mempool_free(), read comment there */
519
-
smp_wmb();
520
-
/*
521
-
* Update the allocation stack trace as this is more useful
522
-
* for debugging.
523
-
*/
524
-
kmemleak_update_trace(element);
525
-
return element;
526
-
}
527
-
spin_unlock_irqrestore(&pool->lock, flags);
528
-
529
-
return NULL;
495
+
return mempool_alloc_from_pool(pool, GFP_NOWAIT);
530
496
}
531
497
EXPORT_SYMBOL(mempool_alloc_preallocated);
532
498