tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm/huge_memory: replace can_split_folio() with direct refcount calculation

can_split_folio() is just a refcount comparison, making sure only the
split caller holds an extra pin. Open code it with
folio_expected_ref_count() != folio_ref_count() - 1. For the extra_pins
used by folio_ref_freeze(), add folio_cache_ref_count() to calculate it.
Also replace folio_expected_ref_count() with folio_cache_ref_count() used
by folio_ref_unfreeze(), since they are returning the same values when a
folio is frozen and folio_cache_ref_count() does not have unnecessary
folio_mapcount() in its implementation.

Link: https://lkml.kernel.org/r/20251126210618.1971206-3-ziy@nvidia.com
Signed-off-by: Zi Yan <ziy@nvidia.com>
Suggested-by: David Hildenbrand (Red Hat) <david@kernel.org>
Reviewed-by: Wei Yang <richard.weiyang@gmail.com>
Acked-by: David Hildenbrand (Red Hat) <david@kernel.org>
Cc: Balbir Singh <balbirs@nvidia.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Naoya Horiguchi <nao.horiguchi@gmail.com>
Cc: Nico Pache <npache@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Zi Yan and committed by
Andrew Morton 5842bcbf bdd0d69a

+22 -34
-1
include/linux/huge_mm.h
··· 369 369 SPLIT_TYPE_NON_UNIFORM, 370 370 }; 371 371 372 - bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins); 373 372 int __split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 374 373 unsigned int new_order); 375 374 int folio_split_unmapped(struct folio *folio, unsigned int new_order);
+20 -32
mm/huge_memory.c
··· 3455 3455 } 3456 3456 } 3457 3457 3458 - /* Racy check whether the huge page can be split */ 3459 - bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) 3460 - { 3461 - int extra_pins; 3462 - 3463 - /* Additional pins from page cache */ 3464 - if (folio_test_anon(folio)) 3465 - extra_pins = folio_test_swapcache(folio) ? 3466 - folio_nr_pages(folio) : 0; 3467 - else 3468 - extra_pins = folio_nr_pages(folio); 3469 - if (pextra_pins) 3470 - *pextra_pins = extra_pins; 3471 - return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 3472 - caller_pins; 3473 - } 3474 - 3475 3458 static bool page_range_has_hwpoisoned(struct page *page, long nr_pages) 3476 3459 { 3477 3460 for (; nr_pages; page++, nr_pages--) ··· 3750 3767 return 0; 3751 3768 } 3752 3769 3770 + /* Number of folio references from the pagecache or the swapcache. */ 3771 + static unsigned int folio_cache_ref_count(const struct folio *folio) 3772 + { 3773 + if (folio_test_anon(folio) && !folio_test_swapcache(folio)) 3774 + return 0; 3775 + return folio_nr_pages(folio); 3776 + } 3777 + 3753 3778 static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int new_order, 3754 3779 struct page *split_at, struct xa_state *xas, 3755 3780 struct address_space *mapping, bool do_lru, 3756 3781 struct list_head *list, enum split_type split_type, 3757 - pgoff_t end, int *nr_shmem_dropped, int extra_pins) 3782 + pgoff_t end, int *nr_shmem_dropped) 3758 3783 { 3759 3784 struct folio *end_folio = folio_next(folio); 3760 3785 struct folio *new_folio, *next; ··· 3773 3782 VM_WARN_ON_ONCE(!mapping && end); 3774 3783 /* Prevent deferred_split_scan() touching ->_refcount */ 3775 3784 ds_queue = folio_split_queue_lock(folio); 3776 - if (folio_ref_freeze(folio, 1 + extra_pins)) { 3785 + if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { 3777 3786 struct swap_cluster_info *ci = NULL; 3778 3787 struct lruvec *lruvec; 3779 - int expected_refs; 3780 3788 3781 3789 if (old_order > 1) { 3782 3790 if (!list_empty(&folio->_deferred_list)) { ··· 3843 3853 3844 3854 zone_device_private_split_cb(folio, new_folio); 3845 3855 3846 - expected_refs = folio_expected_ref_count(new_folio) + 1; 3847 - folio_ref_unfreeze(new_folio, expected_refs); 3856 + folio_ref_unfreeze(new_folio, 3857 + folio_cache_ref_count(new_folio) + 1); 3848 3858 3849 3859 if (do_lru) 3850 3860 lru_add_split_folio(folio, new_folio, lruvec, list); ··· 3887 3897 * Otherwise, a parallel folio_try_get() can grab @folio 3888 3898 * and its caller can see stale page cache entries. 3889 3899 */ 3890 - expected_refs = folio_expected_ref_count(folio) + 1; 3891 - folio_ref_unfreeze(folio, expected_refs); 3900 + folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); 3892 3901 3893 3902 if (do_lru) 3894 3903 unlock_page_lruvec(lruvec); ··· 3936 3947 struct folio *new_folio, *next; 3937 3948 int nr_shmem_dropped = 0; 3938 3949 int remap_flags = 0; 3939 - int extra_pins, ret; 3950 + int ret; 3940 3951 pgoff_t end = 0; 3941 3952 3942 3953 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); ··· 4017 4028 * Racy check if we can split the page, before unmap_folio() will 4018 4029 * split PMDs 4019 4030 */ 4020 - if (!can_split_folio(folio, 1, &extra_pins)) { 4031 + if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) { 4021 4032 ret = -EAGAIN; 4022 4033 goto out_unlock; 4023 4034 } ··· 4040 4051 } 4041 4052 4042 4053 ret = __folio_freeze_and_split_unmapped(folio, new_order, split_at, &xas, mapping, 4043 - true, list, split_type, end, &nr_shmem_dropped, 4044 - extra_pins); 4054 + true, list, split_type, end, &nr_shmem_dropped); 4045 4055 fail: 4046 4056 if (mapping) 4047 4057 xas_unlock(&xas); ··· 4114 4126 */ 4115 4127 int folio_split_unmapped(struct folio *folio, unsigned int new_order) 4116 4128 { 4117 - int extra_pins, ret = 0; 4129 + int ret = 0; 4118 4130 4119 4131 VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio); 4120 4132 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 4121 4133 VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); 4122 4134 VM_WARN_ON_ONCE_FOLIO(!folio_test_anon(folio), folio); 4123 4135 4124 - if (!can_split_folio(folio, 1, &extra_pins)) 4136 + if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) 4125 4137 return -EAGAIN; 4126 4138 4127 4139 local_irq_disable(); 4128 4140 ret = __folio_freeze_and_split_unmapped(folio, new_order, &folio->page, NULL, 4129 4141 NULL, false, NULL, SPLIT_TYPE_UNIFORM, 4130 - 0, NULL, extra_pins); 4142 + 0, NULL); 4131 4143 local_irq_enable(); 4132 4144 return ret; 4133 4145 } ··· 4620 4632 * can be split or not. So skip the check here. 4621 4633 */ 4622 4634 if (!folio_test_private(folio) && 4623 - !can_split_folio(folio, 0, NULL)) 4635 + folio_expected_ref_count(folio) != folio_ref_count(folio)) 4624 4636 goto next; 4625 4637 4626 4638 if (!folio_trylock(folio))
+2 -1
mm/vmscan.c
··· 1284 1284 goto keep_locked; 1285 1285 if (folio_test_large(folio)) { 1286 1286 /* cannot split folio, skip it */ 1287 - if (!can_split_folio(folio, 1, NULL)) 1287 + if (folio_expected_ref_count(folio) != 1288 + folio_ref_count(folio) - 1) 1288 1289 goto activate_locked; 1289 1290 /* 1290 1291 * Split partially mapped folios right away.