mm/mempolicy: add alloc_frozen_pages()

Provide an interface to allocate pages from the page allocator without
incrementing their refcount. This saves an atomic operation on free,
which may be beneficial to some users (eg slab).

Link: https://lkml.kernel.org/r/20241125210149.2976098-15-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: William Kucharski <william.kucharski@oracle.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Matthew Wilcox (Oracle) and committed by

Andrew Morton 1 year ago 64297524 49249a2a

+44 -17

2 changed files

expand all

internal.h

mempolicy.c

+12

mm/internal.h

··· 747 747 void free_frozen_pages(struct page *page, unsigned int order); 748 748 void free_unref_folios(struct folio_batch *fbatch); 749 749 750 + #ifdef CONFIG_NUMA 751 + struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); 752 + #else 753 + static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) 754 + { 755 + return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL); 756 + } 757 + #endif 758 + 759 + #define alloc_frozen_pages(...) \ 760 + alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) 761 + 750 762 extern void zone_pcp_reset(struct zone *zone); 751 763 extern void zone_pcp_disable(struct zone *zone); 752 764 extern void zone_pcp_enable(struct zone *zone);

+32 -17

mm/mempolicy.c

··· 2205 2205 */ 2206 2206 preferred_gfp = gfp | __GFP_NOWARN; 2207 2207 preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); 2208 - page = __alloc_pages_noprof(preferred_gfp, order, nid, nodemask); 2208 + page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask); 2209 2209 if (!page) 2210 - page = __alloc_pages_noprof(gfp, order, nid, NULL); 2210 + page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL); 2211 2211 2212 2212 return page; 2213 2213 } ··· 2253 2253 * First, try to allocate THP only on local node, but 2254 2254 * don't reclaim unnecessarily, just compact. 2255 2255 */ 2256 - page = __alloc_pages_node_noprof(nid, 2257 - gfp | __GFP_THISNODE | __GFP_NORETRY, order); 2256 + page = __alloc_frozen_pages_noprof( 2257 + gfp | __GFP_THISNODE | __GFP_NORETRY, order, 2258 + nid, NULL); 2258 2259 if (page || !(gfp & __GFP_DIRECT_RECLAIM)) 2259 2260 return page; 2260 2261 /* ··· 2267 2266 } 2268 2267 } 2269 2268 2270 - page = __alloc_pages_noprof(gfp, order, nid, nodemask); 2269 + page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask); 2271 2270 2272 2271 if (unlikely(pol->mode == MPOL_INTERLEAVE || 2273 2272 pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) { ··· 2286 2285 struct folio *folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order, 2287 2286 struct mempolicy *pol, pgoff_t ilx, int nid) 2288 2287 { 2289 - return page_rmappable_folio(alloc_pages_mpol(gfp | __GFP_COMP, 2290 - order, pol, ilx, nid)); 2288 + struct page *page = alloc_pages_mpol(gfp | __GFP_COMP, order, pol, 2289 + ilx, nid); 2290 + if (!page) 2291 + return NULL; 2292 + 2293 + set_page_refcounted(page); 2294 + return page_rmappable_folio(page); 2291 2295 } 2292 2296 2293 2297 /** ··· 2327 2321 } 2328 2322 EXPORT_SYMBOL(vma_alloc_folio_noprof); 2329 2323 2324 + struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned order) 2325 + { 2326 + struct mempolicy *pol = &default_policy; 2327 + 2328 + /* 2329 + * No reference counting needed for current->mempolicy 2330 + * nor system default_policy 2331 + */ 2332 + if (!in_interrupt() && !(gfp & __GFP_THISNODE)) 2333 + pol = get_task_policy(current); 2334 + 2335 + return alloc_pages_mpol(gfp, order, pol, NO_INTERLEAVE_INDEX, 2336 + numa_node_id()); 2337 + } 2338 + 2330 2339 /** 2331 2340 * alloc_pages - Allocate pages. 2332 2341 * @gfp: GFP flags. ··· 2358 2337 */ 2359 2338 struct page *alloc_pages_noprof(gfp_t gfp, unsigned int order) 2360 2339 { 2361 - struct mempolicy *pol = &default_policy; 2340 + struct page *page = alloc_frozen_pages_noprof(gfp, order); 2362 2341 2363 - /* 2364 - * No reference counting needed for current->mempolicy 2365 - * nor system default_policy 2366 - */ 2367 - if (!in_interrupt() && !(gfp & __GFP_THISNODE)) 2368 - pol = get_task_policy(current); 2369 - 2370 - return alloc_pages_mpol(gfp, order, pol, NO_INTERLEAVE_INDEX, 2371 - numa_node_id()); 2342 + if (page) 2343 + set_page_refcounted(page); 2344 + return page; 2372 2345 } 2373 2346 EXPORT_SYMBOL(alloc_pages_noprof); 2374 2347

Configure Feed

Configure Feed