tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm/page_alloc: refactor the initial compaction handling

The initial direct compaction done in some cases in
__alloc_pages_slowpath() stands out from the main retry loop of reclaim +
compaction.

We can simplify this by instead skipping the initial reclaim attempt via a
new local variable compact_first, and handle the compact_prority as
necessary to match the original behavior. No functional change intended.

Link: https://lkml.kernel.org/r/20260106-thp-thisnode-tweak-v3-2-f5d67c21a193@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Suggested-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Joshua Hahn <joshua.hahnjy@gmail.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: David Hildenbrand (Red Hat) <david@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Vlastimil Babka and committed by
Andrew Morton 53a9b464 66987218

+55 -53
+7 -1
include/linux/gfp.h
··· 407 407 /* Returns true if the gfp_mask allows use of ALLOC_NO_WATERMARK */ 408 408 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask); 409 409 410 + /* A helper for checking if gfp includes all the specified flags */ 411 + static inline bool gfp_has_flags(gfp_t gfp, gfp_t flags) 412 + { 413 + return (gfp & flags) == flags; 414 + } 415 + 410 416 static inline bool gfp_has_io_fs(gfp_t gfp) 411 417 { 412 - return (gfp & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS); 418 + return gfp_has_flags(gfp, __GFP_IO | __GFP_FS); 413 419 } 414 420 415 421 /*
+48 -52
mm/page_alloc.c
··· 4694 4694 struct alloc_context *ac) 4695 4695 { 4696 4696 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4697 - bool can_compact = gfp_compaction_allowed(gfp_mask); 4697 + bool can_compact = can_direct_reclaim && gfp_compaction_allowed(gfp_mask); 4698 4698 bool nofail = gfp_mask & __GFP_NOFAIL; 4699 4699 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4700 4700 struct page *page = NULL; ··· 4707 4707 unsigned int cpuset_mems_cookie; 4708 4708 unsigned int zonelist_iter_cookie; 4709 4709 int reserve_flags; 4710 + bool compact_first = false; 4710 4711 4711 4712 if (unlikely(nofail)) { 4712 4713 /* ··· 4730 4729 compact_priority = DEF_COMPACT_PRIORITY; 4731 4730 cpuset_mems_cookie = read_mems_allowed_begin(); 4732 4731 zonelist_iter_cookie = zonelist_iter_begin(); 4732 + 4733 + /* 4734 + * For costly allocations, try direct compaction first, as it's likely 4735 + * that we have enough base pages and don't need to reclaim. For non- 4736 + * movable high-order allocations, do that as well, as compaction will 4737 + * try prevent permanent fragmentation by migrating from blocks of the 4738 + * same migratetype. 4739 + */ 4740 + if (can_compact && (costly_order || (order > 0 && 4741 + ac->migratetype != MIGRATE_MOVABLE))) { 4742 + compact_first = true; 4743 + compact_priority = INIT_COMPACT_PRIORITY; 4744 + } 4733 4745 4734 4746 /* 4735 4747 * The fast path uses conservative alloc_flags to succeed only until ··· 4785 4771 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4786 4772 if (page) 4787 4773 goto got_pg; 4788 - 4789 - /* 4790 - * For costly allocations, try direct compaction first, as it's likely 4791 - * that we have enough base pages and don't need to reclaim. For non- 4792 - * movable high-order allocations, do that as well, as compaction will 4793 - * try prevent permanent fragmentation by migrating from blocks of the 4794 - * same migratetype. 4795 - * Don't try this for allocations that are allowed to ignore 4796 - * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4797 - */ 4798 - if (can_direct_reclaim && can_compact && 4799 - (costly_order || 4800 - (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4801 - && !gfp_pfmemalloc_allowed(gfp_mask)) { 4802 - page = __alloc_pages_direct_compact(gfp_mask, order, 4803 - alloc_flags, ac, 4804 - INIT_COMPACT_PRIORITY, 4805 - &compact_result); 4806 - if (page) 4807 - goto got_pg; 4808 - 4809 - /* 4810 - * Checks for costly allocations with __GFP_NORETRY, which 4811 - * includes some THP page fault allocations 4812 - */ 4813 - if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4814 - /* 4815 - * THP page faults may attempt local node only first, 4816 - * but are then allowed to only compact, not reclaim, 4817 - * see alloc_pages_mpol(). 4818 - * 4819 - * Compaction has failed above and we don't want such 4820 - * THP allocations to put reclaim pressure on a single 4821 - * node in a situation where other nodes might have 4822 - * plenty of available memory. 4823 - */ 4824 - if (gfp_mask & __GFP_THISNODE) 4825 - goto nopage; 4826 - 4827 - /* 4828 - * Proceed with single round of reclaim/compaction, but 4829 - * since sync compaction could be very expensive, keep 4830 - * using async compaction. 4831 - */ 4832 - compact_priority = INIT_COMPACT_PRIORITY; 4833 - } 4834 - } 4835 4774 4836 4775 retry: 4837 4776 /* ··· 4829 4862 goto nopage; 4830 4863 4831 4864 /* Try direct reclaim and then allocating */ 4832 - page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4833 - &did_some_progress); 4834 - if (page) 4835 - goto got_pg; 4865 + if (!compact_first) { 4866 + page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, 4867 + ac, &did_some_progress); 4868 + if (page) 4869 + goto got_pg; 4870 + } 4836 4871 4837 4872 /* Try direct compaction and then allocating */ 4838 4873 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4839 4874 compact_priority, &compact_result); 4840 4875 if (page) 4841 4876 goto got_pg; 4877 + 4878 + if (compact_first) { 4879 + /* 4880 + * THP page faults may attempt local node only first, but are 4881 + * then allowed to only compact, not reclaim, see 4882 + * alloc_pages_mpol(). 4883 + * 4884 + * Compaction has failed above and we don't want such THP 4885 + * allocations to put reclaim pressure on a single node in a 4886 + * situation where other nodes might have plenty of available 4887 + * memory. 4888 + */ 4889 + if (gfp_has_flags(gfp_mask, __GFP_NORETRY | __GFP_THISNODE)) 4890 + goto nopage; 4891 + 4892 + /* 4893 + * For the initial compaction attempt we have lowered its 4894 + * priority. Restore it for further retries, if those are 4895 + * allowed. With __GFP_NORETRY there will be a single round of 4896 + * reclaim and compaction with the lowered priority. 4897 + */ 4898 + if (!(gfp_mask & __GFP_NORETRY)) 4899 + compact_priority = DEF_COMPACT_PRIORITY; 4900 + 4901 + compact_first = false; 4902 + goto retry; 4903 + } 4842 4904 4843 4905 /* Do not loop if specifically requested */ 4844 4906 if (gfp_mask & __GFP_NORETRY)