mm: page_alloc: add alloc_contig_frozen_{range,pages}()

+19 -29

include/linux/gfp.h

··· 436 436 #define ACR_FLAGS_CMA ((__force acr_flags_t)BIT(0)) // allocate for CMA 437 437 438 438 /* The below functions must be run on a range from a single zone. */ 439 - extern int alloc_contig_range_noprof(unsigned long start, unsigned long end, 440 - acr_flags_t alloc_flags, gfp_t gfp_mask); 441 - #define alloc_contig_range(...) alloc_hooks(alloc_contig_range_noprof(__VA_ARGS__)) 439 + int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end, 440 + acr_flags_t alloc_flags, gfp_t gfp_mask); 441 + #define alloc_contig_frozen_range(...) \ 442 + alloc_hooks(alloc_contig_frozen_range_noprof(__VA_ARGS__)) 442 443 443 - extern struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, 444 - int nid, nodemask_t *nodemask); 445 - #define alloc_contig_pages(...) alloc_hooks(alloc_contig_pages_noprof(__VA_ARGS__)) 444 + int alloc_contig_range_noprof(unsigned long start, unsigned long end, 445 + acr_flags_t alloc_flags, gfp_t gfp_mask); 446 + #define alloc_contig_range(...) \ 447 + alloc_hooks(alloc_contig_range_noprof(__VA_ARGS__)) 446 448 449 + struct page *alloc_contig_frozen_pages_noprof(unsigned long nr_pages, 450 + gfp_t gfp_mask, int nid, nodemask_t *nodemask); 451 + #define alloc_contig_frozen_pages(...) \ 452 + alloc_hooks(alloc_contig_frozen_pages_noprof(__VA_ARGS__)) 453 + 454 + struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, 455 + int nid, nodemask_t *nodemask); 456 + #define alloc_contig_pages(...) \ 457 + alloc_hooks(alloc_contig_pages_noprof(__VA_ARGS__)) 458 + 459 + void free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages); 447 460 void free_contig_range(unsigned long pfn, unsigned long nr_pages); 448 461 #endif 449 - 450 - #ifdef CONFIG_CONTIG_ALLOC 451 - static inline struct folio *folio_alloc_gigantic_noprof(int order, gfp_t gfp, 452 - int nid, nodemask_t *node) 453 - { 454 - struct page *page; 455 - 456 - if (WARN_ON(!order || !(gfp & __GFP_COMP))) 457 - return NULL; 458 - 459 - page = alloc_contig_pages_noprof(1 << order, gfp, nid, node); 460 - 461 - return page ? page_folio(page) : NULL; 462 - } 463 - #else 464 - static inline struct folio *folio_alloc_gigantic_noprof(int order, gfp_t gfp, 465 - int nid, nodemask_t *node) 466 - { 467 - return NULL; 468 - } 469 - #endif 470 - /* This should be paired with folio_put() rather than free_contig_range(). */ 471 - #define folio_alloc_gigantic(...) alloc_hooks(folio_alloc_gigantic_noprof(__VA_ARGS__)) 472 462 473 463 DEFINE_FREE(free_page, void *, free_page((unsigned long)_T)) 474 464

+7 -2

mm/cma.c

··· 836 836 spin_unlock_irq(&cma->lock); 837 837 838 838 mutex_lock(&cma->alloc_mutex); 839 - ret = alloc_contig_range(pfn, pfn + count, ACR_FLAGS_CMA, gfp); 839 + ret = alloc_contig_frozen_range(pfn, pfn + count, ACR_FLAGS_CMA, gfp); 840 840 mutex_unlock(&cma->alloc_mutex); 841 841 if (!ret) 842 842 break; ··· 904 904 trace_cma_alloc_finish(name, page ? page_to_pfn(page) : 0, 905 905 page, count, align, ret); 906 906 if (page) { 907 + set_pages_refcounted(page, count); 907 908 count_vm_event(CMA_ALLOC_SUCCESS); 908 909 cma_sysfs_account_success_pages(cma, count); 909 910 } else { ··· 984 983 return false; 985 984 } 986 985 987 - free_contig_range(pfn, count); 986 + if (PageHead(pages)) 987 + __free_pages((struct page *)pages, compound_order(pages)); 988 + else 989 + free_contig_range(pfn, count); 990 + 988 991 cma_clear_bitmap(cma, cmr, pfn, count); 989 992 cma_sysfs_account_release_pages(cma, count); 990 993 trace_cma_release(cma->name, pfn, pages, count);

+7 -2

mm/hugetlb.c

··· 1428 1428 retry: 1429 1429 folio = hugetlb_cma_alloc_folio(order, gfp_mask, nid, nodemask); 1430 1430 if (!folio) { 1431 + struct page *page; 1432 + 1431 1433 if (hugetlb_cma_exclusive_alloc()) 1432 1434 return NULL; 1433 1435 1434 - folio = folio_alloc_gigantic(order, gfp_mask, nid, nodemask); 1435 - if (!folio) 1436 + page = alloc_contig_frozen_pages(1 << order, gfp_mask, nid, nodemask); 1437 + if (!page) 1436 1438 return NULL; 1439 + 1440 + set_page_refcounted(page); 1441 + folio = page_folio(page); 1437 1442 } 1438 1443 1439 1444 if (folio_ref_freeze(folio, 1))

+13

mm/internal.h

··· 580 580 set_page_count(page, 1); 581 581 } 582 582 583 + static inline void set_pages_refcounted(struct page *page, unsigned long nr_pages) 584 + { 585 + unsigned long pfn = page_to_pfn(page); 586 + 587 + if (PageHead(page)) { 588 + set_page_refcounted(page); 589 + return; 590 + } 591 + 592 + for (; nr_pages--; pfn++) 593 + set_page_refcounted(pfn_to_page(pfn)); 594 + } 595 + 583 596 /* 584 597 * Return true if a folio needs ->release_folio() calling upon it. 585 598 */

+136 -50

mm/page_alloc.c

··· 6889 6889 return (ret < 0) ? ret : 0; 6890 6890 } 6891 6891 6892 - static void split_free_pages(struct list_head *list, gfp_t gfp_mask) 6892 + static void split_free_frozen_pages(struct list_head *list, gfp_t gfp_mask) 6893 6893 { 6894 6894 int order; 6895 6895 ··· 6901 6901 int i; 6902 6902 6903 6903 post_alloc_hook(page, order, gfp_mask); 6904 - set_page_refcounted(page); 6905 6904 if (!order) 6906 6905 continue; 6907 6906 6908 - split_page(page, order); 6907 + __split_page(page, order); 6909 6908 6910 6909 /* Add all subpages to the order-0 head, in sequence. */ 6911 6910 list_del(&page->lru); ··· 6948 6949 return 0; 6949 6950 } 6950 6951 6952 + static void __free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages) 6953 + { 6954 + for (; nr_pages--; pfn++) 6955 + free_frozen_pages(pfn_to_page(pfn), 0); 6956 + } 6957 + 6951 6958 /** 6952 - * alloc_contig_range() -- tries to allocate given range of pages 6959 + * alloc_contig_frozen_range() -- tries to allocate given range of frozen pages 6953 6960 * @start: start PFN to allocate 6954 6961 * @end: one-past-the-last PFN to allocate 6955 6962 * @alloc_flags: allocation information ··· 6970 6965 * pageblocks in the range. Once isolated, the pageblocks should not 6971 6966 * be modified by others. 6972 6967 * 6973 - * Return: zero on success or negative error code. On success all 6974 - * pages which PFN is in [start, end) are allocated for the caller and 6975 - * need to be freed with free_contig_range(). 6968 + * All frozen pages which PFN is in [start, end) are allocated for the 6969 + * caller, and they could be freed with free_contig_frozen_range(), 6970 + * free_frozen_pages() also could be used to free compound frozen pages 6971 + * directly. 6972 + * 6973 + * Return: zero on success or negative error code. 6976 6974 */ 6977 - int alloc_contig_range_noprof(unsigned long start, unsigned long end, 6978 - acr_flags_t alloc_flags, gfp_t gfp_mask) 6975 + int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end, 6976 + acr_flags_t alloc_flags, gfp_t gfp_mask) 6979 6977 { 6980 6978 const unsigned int order = ilog2(end - start); 6981 6979 unsigned long outer_start, outer_end; ··· 7094 7086 } 7095 7087 7096 7088 if (!(gfp_mask & __GFP_COMP)) { 7097 - split_free_pages(cc.freepages, gfp_mask); 7089 + split_free_frozen_pages(cc.freepages, gfp_mask); 7098 7090 7099 7091 /* Free head and tail (if any) */ 7100 7092 if (start != outer_start) 7101 - free_contig_range(outer_start, start - outer_start); 7093 + __free_contig_frozen_range(outer_start, start - outer_start); 7102 7094 if (end != outer_end) 7103 - free_contig_range(end, outer_end - end); 7095 + __free_contig_frozen_range(end, outer_end - end); 7104 7096 } else if (start == outer_start && end == outer_end && is_power_of_2(end - start)) { 7105 7097 struct page *head = pfn_to_page(start); 7106 7098 7107 7099 check_new_pages(head, order); 7108 7100 prep_new_page(head, order, gfp_mask, 0); 7109 - set_page_refcounted(head); 7110 7101 } else { 7111 7102 ret = -EINVAL; 7112 7103 WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n", ··· 7115 7108 undo_isolate_page_range(start, end); 7116 7109 return ret; 7117 7110 } 7118 - EXPORT_SYMBOL(alloc_contig_range_noprof); 7111 + EXPORT_SYMBOL(alloc_contig_frozen_range_noprof); 7119 7112 7120 - static int __alloc_contig_pages(unsigned long start_pfn, 7121 - unsigned long nr_pages, gfp_t gfp_mask) 7113 + /** 7114 + * alloc_contig_range() -- tries to allocate given range of pages 7115 + * @start: start PFN to allocate 7116 + * @end: one-past-the-last PFN to allocate 7117 + * @alloc_flags: allocation information 7118 + * @gfp_mask: GFP mask. 7119 + * 7120 + * This routine is a wrapper around alloc_contig_frozen_range(), it can't 7121 + * be used to allocate compound pages, the refcount of each allocated page 7122 + * will be set to one. 7123 + * 7124 + * All pages which PFN is in [start, end) are allocated for the caller, 7125 + * and should be freed with free_contig_range() or by manually calling 7126 + * __free_page() on each allocated page. 7127 + * 7128 + * Return: zero on success or negative error code. 7129 + */ 7130 + int alloc_contig_range_noprof(unsigned long start, unsigned long end, 7131 + acr_flags_t alloc_flags, gfp_t gfp_mask) 7122 7132 { 7123 - unsigned long end_pfn = start_pfn + nr_pages; 7133 + int ret; 7124 7134 7125 - return alloc_contig_range_noprof(start_pfn, end_pfn, ACR_FLAGS_NONE, 7126 - gfp_mask); 7135 + if (WARN_ON(gfp_mask & __GFP_COMP)) 7136 + return -EINVAL; 7137 + 7138 + ret = alloc_contig_frozen_range_noprof(start, end, alloc_flags, gfp_mask); 7139 + if (!ret) 7140 + set_pages_refcounted(pfn_to_page(start), end - start); 7141 + 7142 + return ret; 7127 7143 } 7144 + EXPORT_SYMBOL(alloc_contig_range_noprof); 7128 7145 7129 7146 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 7130 7147 unsigned long nr_pages, bool skip_hugetlb, ··· 7217 7186 } 7218 7187 7219 7188 /** 7220 - * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 7189 + * alloc_contig_frozen_pages() -- tries to find and allocate contiguous range of frozen pages 7221 7190 * @nr_pages: Number of contiguous pages to allocate 7222 7191 * @gfp_mask: GFP mask. Node/zone/placement hints limit the search; only some 7223 7192 * action and reclaim modifiers are supported. Reclaim modifiers ··· 7225 7194 * @nid: Target node 7226 7195 * @nodemask: Mask for other possible nodes 7227 7196 * 7228 - * This routine is a wrapper around alloc_contig_range(). It scans over zones 7229 - * on an applicable zonelist to find a contiguous pfn range which can then be 7230 - * tried for allocation with alloc_contig_range(). This routine is intended 7231 - * for allocation requests which can not be fulfilled with the buddy allocator. 7197 + * This routine is a wrapper around alloc_contig_frozen_range(). It scans over 7198 + * zones on an applicable zonelist to find a contiguous pfn range which can then 7199 + * be tried for allocation with alloc_contig_frozen_range(). This routine is 7200 + * intended for allocation requests which can not be fulfilled with the buddy 7201 + * allocator. 7232 7202 * 7233 7203 * The allocated memory is always aligned to a page boundary. If nr_pages is a 7234 7204 * power of two, then allocated range is also guaranteed to be aligned to same 7235 7205 * nr_pages (e.g. 1GB request would be aligned to 1GB). 7236 7206 * 7237 - * Allocated pages can be freed with free_contig_range() or by manually calling 7238 - * __free_page() on each allocated page. 7207 + * Allocated frozen pages need be freed with free_contig_frozen_range(), 7208 + * or by manually calling free_frozen_pages() on each allocated frozen 7209 + * non-compound page, for compound frozen pages could be freed with 7210 + * free_frozen_pages() directly. 7239 7211 * 7240 - * Return: pointer to contiguous pages on success, or NULL if not successful. 7212 + * Return: pointer to contiguous frozen pages on success, or NULL if not successful. 7241 7213 */ 7242 - struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, 7243 - int nid, nodemask_t *nodemask) 7214 + struct page *alloc_contig_frozen_pages_noprof(unsigned long nr_pages, 7215 + gfp_t gfp_mask, int nid, nodemask_t *nodemask) 7244 7216 { 7245 7217 unsigned long ret, pfn, flags; 7246 7218 struct zonelist *zonelist; ··· 7265 7231 &skipped_hugetlb)) { 7266 7232 /* 7267 7233 * We release the zone lock here because 7268 - * alloc_contig_range() will also lock the zone 7269 - * at some point. If there's an allocation 7270 - * spinning on this lock, it may win the race 7271 - * and cause alloc_contig_range() to fail... 7234 + * alloc_contig_frozen_range() will also lock 7235 + * the zone at some point. If there's an 7236 + * allocation spinning on this lock, it may 7237 + * win the race and cause allocation to fail. 7272 7238 */ 7273 7239 spin_unlock_irqrestore(&zone->lock, flags); 7274 - ret = __alloc_contig_pages(pfn, nr_pages, 7240 + ret = alloc_contig_frozen_range_noprof(pfn, 7241 + pfn + nr_pages, 7242 + ACR_FLAGS_NONE, 7275 7243 gfp_mask); 7276 7244 if (!ret) 7277 7245 return pfn_to_page(pfn); ··· 7296 7260 } 7297 7261 return NULL; 7298 7262 } 7263 + EXPORT_SYMBOL(alloc_contig_frozen_pages_noprof); 7299 7264 7300 - void free_contig_range(unsigned long pfn, unsigned long nr_pages) 7265 + /** 7266 + * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 7267 + * @nr_pages: Number of contiguous pages to allocate 7268 + * @gfp_mask: GFP mask. 7269 + * @nid: Target node 7270 + * @nodemask: Mask for other possible nodes 7271 + * 7272 + * This routine is a wrapper around alloc_contig_frozen_pages(), it can't 7273 + * be used to allocate compound pages, the refcount of each allocated page 7274 + * will be set to one. 7275 + * 7276 + * Allocated pages can be freed with free_contig_range() or by manually 7277 + * calling __free_page() on each allocated page. 7278 + * 7279 + * Return: pointer to contiguous pages on success, or NULL if not successful. 7280 + */ 7281 + struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, 7282 + int nid, nodemask_t *nodemask) 7301 7283 { 7302 - unsigned long count = 0; 7303 - struct folio *folio = pfn_folio(pfn); 7284 + struct page *page; 7304 7285 7305 - if (folio_test_large(folio)) { 7306 - int expected = folio_nr_pages(folio); 7286 + if (WARN_ON(gfp_mask & __GFP_COMP)) 7287 + return NULL; 7307 7288 7308 - if (nr_pages == expected) 7309 - folio_put(folio); 7310 - else 7311 - WARN(true, "PFN %lu: nr_pages %lu != expected %d\n", 7312 - pfn, nr_pages, expected); 7289 + page = alloc_contig_frozen_pages_noprof(nr_pages, gfp_mask, nid, 7290 + nodemask); 7291 + if (page) 7292 + set_pages_refcounted(page, nr_pages); 7293 + 7294 + return page; 7295 + } 7296 + EXPORT_SYMBOL(alloc_contig_pages_noprof); 7297 + 7298 + /** 7299 + * free_contig_frozen_range() -- free the contiguous range of frozen pages 7300 + * @pfn: start PFN to free 7301 + * @nr_pages: Number of contiguous frozen pages to free 7302 + * 7303 + * This can be used to free the allocated compound/non-compound frozen pages. 7304 + */ 7305 + void free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages) 7306 + { 7307 + struct page *first_page = pfn_to_page(pfn); 7308 + const unsigned int order = ilog2(nr_pages); 7309 + 7310 + if (WARN_ON_ONCE(first_page != compound_head(first_page))) 7311 + return; 7312 + 7313 + if (PageHead(first_page)) { 7314 + WARN_ON_ONCE(order != compound_order(first_page)); 7315 + free_frozen_pages(first_page, order); 7313 7316 return; 7314 7317 } 7315 7318 7316 - for (; nr_pages--; pfn++) { 7317 - struct page *page = pfn_to_page(pfn); 7319 + __free_contig_frozen_range(pfn, nr_pages); 7320 + } 7321 + EXPORT_SYMBOL(free_contig_frozen_range); 7318 7322 7319 - count += page_count(page) != 1; 7320 - __free_page(page); 7321 - } 7322 - WARN(count != 0, "%lu pages are still in use!\n", count); 7323 + /** 7324 + * free_contig_range() -- free the contiguous range of pages 7325 + * @pfn: start PFN to free 7326 + * @nr_pages: Number of contiguous pages to free 7327 + * 7328 + * This can be only used to free the allocated non-compound pages. 7329 + */ 7330 + void free_contig_range(unsigned long pfn, unsigned long nr_pages) 7331 + { 7332 + if (WARN_ON_ONCE(PageHead(pfn_to_page(pfn)))) 7333 + return; 7334 + 7335 + for (; nr_pages--; pfn++) 7336 + __free_page(pfn_to_page(pfn)); 7323 7337 } 7324 7338 EXPORT_SYMBOL(free_contig_range); 7325 7339 #endif /* CONFIG_CONTIG_ALLOC */

Configure Feed

Configure Feed