tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm: hugetlb: allocate frozen pages for gigantic allocation

alloc_gigantic_folio() allocates a folio with refcount increated and then
freeze it, convert to allocate a frozen folio to remove the atomic
operation about folio refcount, and saving atomic operation during
__update_and_free_hugetlb_folio() too.

Besides, rename hugetlb_cma_{alloc,free}_folio(), alloc_gigantic_folio()
and alloc_buddy_hugetlb_folio() with frozen which make them more
self-explanatory.

Link: https://lkml.kernel.org/r/20260109093136.1491549-7-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Muchun Song <muchun.song@linux.dev>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Jane Chu <jane.chu@oracle.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Claudiu Beznea <claudiu.beznea.uj@bp.renesas.com>
Cc: Mark Brown <broonie@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Kefeng Wang and committed by
Andrew Morton 14f27076 9bda131c

+28 -66
+20 -55
mm/hugetlb.c
··· 121 121 unsigned long start, unsigned long end, bool take_locks); 122 122 static struct resv_map *vma_resv_map(struct vm_area_struct *vma); 123 123 124 - static void hugetlb_free_folio(struct folio *folio) 125 - { 126 - if (folio_test_hugetlb_cma(folio)) { 127 - hugetlb_cma_free_folio(folio); 128 - return; 129 - } 130 - 131 - folio_put(folio); 132 - } 133 - 134 124 static inline bool subpool_is_free(struct hugepage_subpool *spool) 135 125 { 136 126 if (spool->count) ··· 1407 1417 return NULL; 1408 1418 } 1409 1419 1410 - #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE 1411 - #ifdef CONFIG_CONTIG_ALLOC 1412 - static struct folio *alloc_gigantic_folio(int order, gfp_t gfp_mask, 1420 + #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC) 1421 + static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask, 1413 1422 int nid, nodemask_t *nodemask) 1414 1423 { 1415 1424 struct folio *folio; 1416 - bool retried = false; 1417 1425 1418 - retry: 1419 - folio = hugetlb_cma_alloc_folio(order, gfp_mask, nid, nodemask); 1420 - if (!folio) { 1421 - struct page *page; 1422 - 1423 - if (hugetlb_cma_exclusive_alloc()) 1424 - return NULL; 1425 - 1426 - page = alloc_contig_frozen_pages(1 << order, gfp_mask, nid, nodemask); 1427 - if (!page) 1428 - return NULL; 1429 - 1430 - set_page_refcounted(page); 1431 - folio = page_folio(page); 1432 - } 1433 - 1434 - if (folio_ref_freeze(folio, 1)) 1426 + folio = hugetlb_cma_alloc_frozen_folio(order, gfp_mask, nid, nodemask); 1427 + if (folio) 1435 1428 return folio; 1436 1429 1437 - pr_warn("HugeTLB: unexpected refcount on PFN %lu\n", folio_pfn(folio)); 1438 - hugetlb_free_folio(folio); 1439 - if (!retried) { 1440 - retried = true; 1441 - goto retry; 1442 - } 1443 - return NULL; 1444 - } 1430 + if (hugetlb_cma_exclusive_alloc()) 1431 + return NULL; 1445 1432 1446 - #else /* !CONFIG_CONTIG_ALLOC */ 1447 - static struct folio *alloc_gigantic_folio(int order, gfp_t gfp_mask, int nid, 1448 - nodemask_t *nodemask) 1449 - { 1450 - return NULL; 1433 + folio = (struct folio *)alloc_contig_frozen_pages(1 << order, gfp_mask, 1434 + nid, nodemask); 1435 + return folio; 1451 1436 } 1452 - #endif /* CONFIG_CONTIG_ALLOC */ 1453 - 1454 - #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */ 1455 - static struct folio *alloc_gigantic_folio(int order, gfp_t gfp_mask, int nid, 1437 + #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE || !CONFIG_CONTIG_ALLOC */ 1438 + static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask, int nid, 1456 1439 nodemask_t *nodemask) 1457 1440 { 1458 1441 return NULL; ··· 1555 1592 if (unlikely(folio_test_hwpoison(folio))) 1556 1593 folio_clear_hugetlb_hwpoison(folio); 1557 1594 1558 - folio_ref_unfreeze(folio, 1); 1559 - 1560 - hugetlb_free_folio(folio); 1595 + VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); 1596 + if (folio_test_hugetlb_cma(folio)) 1597 + hugetlb_cma_free_frozen_folio(folio); 1598 + else 1599 + free_frozen_pages(&folio->page, folio_order(folio)); 1561 1600 } 1562 1601 1563 1602 /* ··· 1839 1874 return NULL; 1840 1875 } 1841 1876 1842 - static struct folio *alloc_buddy_hugetlb_folio(int order, gfp_t gfp_mask, 1877 + static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask, 1843 1878 int nid, nodemask_t *nmask, nodemask_t *node_alloc_noretry) 1844 1879 { 1845 1880 struct folio *folio; ··· 1895 1930 nid = numa_mem_id(); 1896 1931 1897 1932 if (order_is_gigantic(order)) 1898 - folio = alloc_gigantic_folio(order, gfp_mask, nid, nmask); 1933 + folio = alloc_gigantic_frozen_folio(order, gfp_mask, nid, nmask); 1899 1934 else 1900 - folio = alloc_buddy_hugetlb_folio(order, gfp_mask, nid, nmask, 1901 - node_alloc_noretry); 1935 + folio = alloc_buddy_frozen_folio(order, gfp_mask, nid, nmask, 1936 + node_alloc_noretry); 1902 1937 if (folio) 1903 1938 init_new_hugetlb_folio(folio); 1904 1939 return folio;
+3 -6
mm/hugetlb_cma.c
··· 18 18 static bool hugetlb_cma_only; 19 19 static unsigned long hugetlb_cma_size __initdata; 20 20 21 - void hugetlb_cma_free_folio(struct folio *folio) 21 + void hugetlb_cma_free_frozen_folio(struct folio *folio) 22 22 { 23 - folio_ref_dec(folio); 24 - 25 23 WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)], 26 24 &folio->page, folio_nr_pages(folio))); 27 25 } 28 26 29 - struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask, 30 - int nid, nodemask_t *nodemask) 27 + struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, 28 + int nid, nodemask_t *nodemask) 31 29 { 32 30 int node; 33 31 struct folio *folio; ··· 48 50 if (!page) 49 51 return NULL; 50 52 51 - set_page_refcounted(page); 52 53 folio = page_folio(page); 53 54 folio_set_hugetlb_cma(folio); 54 55 return folio;
+5 -5
mm/hugetlb_cma.h
··· 3 3 #define _LINUX_HUGETLB_CMA_H 4 4 5 5 #ifdef CONFIG_CMA 6 - void hugetlb_cma_free_folio(struct folio *folio); 7 - struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask, 6 + void hugetlb_cma_free_frozen_folio(struct folio *folio); 7 + struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, 8 8 int nid, nodemask_t *nodemask); 9 9 struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, 10 10 bool node_exact); ··· 13 13 void hugetlb_cma_validate_params(void); 14 14 bool hugetlb_early_cma(struct hstate *h); 15 15 #else 16 - static inline void hugetlb_cma_free_folio(struct folio *folio) 16 + static inline void hugetlb_cma_free_frozen_folio(struct folio *folio) 17 17 { 18 18 } 19 19 20 - static inline struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask, 21 - int nid, nodemask_t *nodemask) 20 + static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order, 21 + gfp_t gfp_mask, int nid, nodemask_t *nodemask) 22 22 { 23 23 return NULL; 24 24 }