tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm/huge_memory: implement device-private THP splitting

Add support for splitting device-private THP folios, enabling fallback
to smaller page sizes when large page allocation or migration fails.

Key changes:
- split_huge_pmd(): Handle device-private PMD entries during splitting
- Preserve RMAP_EXCLUSIVE semantics for anonymous exclusive folios
- Skip RMP_USE_SHARED_ZEROPAGE for device-private entries as they
don't support shared zero page semantics

Link: https://lkml.kernel.org/r/20251001065707.920170-6-balbirs@nvidia.com
Signed-off-by: Balbir Singh <balbirs@nvidia.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: Ying Huang <ying.huang@linux.alibaba.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: "Liam R. Howlett" <Liam.Howlett@oracle.com>
Cc: Nico Pache <npache@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Lyude Paul <lyude@redhat.com>
Cc: Danilo Krummrich <dakr@kernel.org>
Cc: David Airlie <airlied@gmail.com>
Cc: Simona Vetter <simona@ffwll.ch>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Cc: Mika Penttilä <mpenttil@redhat.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Francois Dugast <francois.dugast@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Balbir Singh and committed by
Andrew Morton 14628729 65edfda6

+76 -12
+75 -12
mm/huge_memory.c
··· 2842 2842 struct page *page; 2843 2843 pgtable_t pgtable; 2844 2844 pmd_t old_pmd, _pmd; 2845 - bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 2845 + bool soft_dirty, uffd_wp = false, young = false, write = false; 2846 2846 bool anon_exclusive = false, dirty = false; 2847 2847 unsigned long addr; 2848 2848 pte_t *pte; 2849 2849 int i; 2850 + swp_entry_t entry; 2850 2851 2851 2852 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2852 2853 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2853 2854 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2854 - VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)); 2855 + 2856 + VM_WARN_ON(!is_pmd_non_present_folio_entry(*pmd) && !pmd_trans_huge(*pmd)); 2855 2857 2856 2858 count_vm_event(THP_SPLIT_PMD); 2857 2859 ··· 2901 2899 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2902 2900 } 2903 2901 2904 - pmd_migration = is_pmd_migration_entry(*pmd); 2905 - if (unlikely(pmd_migration)) { 2906 - swp_entry_t entry; 2907 2902 2903 + if (is_pmd_migration_entry(*pmd)) { 2908 2904 old_pmd = *pmd; 2909 2905 entry = pmd_to_swp_entry(old_pmd); 2910 2906 page = pfn_swap_entry_to_page(entry); 2907 + folio = page_folio(page); 2908 + 2909 + soft_dirty = pmd_swp_soft_dirty(old_pmd); 2910 + uffd_wp = pmd_swp_uffd_wp(old_pmd); 2911 + 2911 2912 write = is_writable_migration_entry(entry); 2912 2913 if (PageAnon(page)) 2913 2914 anon_exclusive = is_readable_exclusive_migration_entry(entry); 2914 2915 young = is_migration_entry_young(entry); 2915 2916 dirty = is_migration_entry_dirty(entry); 2917 + } else if (is_pmd_device_private_entry(*pmd)) { 2918 + old_pmd = *pmd; 2919 + entry = pmd_to_swp_entry(old_pmd); 2920 + page = pfn_swap_entry_to_page(entry); 2921 + folio = page_folio(page); 2922 + 2916 2923 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2917 2924 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2925 + 2926 + write = is_writable_device_private_entry(entry); 2927 + anon_exclusive = PageAnonExclusive(page); 2928 + 2929 + /* 2930 + * Device private THP should be treated the same as regular 2931 + * folios w.r.t anon exclusive handling. See the comments for 2932 + * folio handling and anon_exclusive below. 2933 + */ 2934 + if (freeze && anon_exclusive && 2935 + folio_try_share_anon_rmap_pmd(folio, page)) 2936 + freeze = false; 2937 + if (!freeze) { 2938 + rmap_t rmap_flags = RMAP_NONE; 2939 + 2940 + folio_ref_add(folio, HPAGE_PMD_NR - 1); 2941 + if (anon_exclusive) 2942 + rmap_flags |= RMAP_EXCLUSIVE; 2943 + 2944 + folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 2945 + vma, haddr, rmap_flags); 2946 + } 2918 2947 } else { 2919 2948 /* 2920 2949 * Up to this point the pmd is present and huge and userland has ··· 3029 2996 * Note that NUMA hinting access restrictions are not transferred to 3030 2997 * avoid any possibility of altering permissions across VMAs. 3031 2998 */ 3032 - if (freeze || pmd_migration) { 3033 - for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 3034 - pte_t entry; 3035 - swp_entry_t swp_entry; 2999 + if (freeze || is_pmd_migration_entry(old_pmd)) { 3000 + pte_t entry; 3001 + swp_entry_t swp_entry; 3036 3002 3003 + for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 3037 3004 if (write) 3038 3005 swp_entry = make_writable_migration_entry( 3039 3006 page_to_pfn(page + i)); ··· 3052 3019 entry = pte_swp_mksoft_dirty(entry); 3053 3020 if (uffd_wp) 3054 3021 entry = pte_swp_mkuffd_wp(entry); 3022 + VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3023 + set_pte_at(mm, addr, pte + i, entry); 3024 + } 3025 + } else if (is_pmd_device_private_entry(old_pmd)) { 3026 + pte_t entry; 3027 + swp_entry_t swp_entry; 3055 3028 3029 + for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 3030 + /* 3031 + * anon_exclusive was already propagated to the relevant 3032 + * pages corresponding to the pte entries when freeze 3033 + * is false. 3034 + */ 3035 + if (write) 3036 + swp_entry = make_writable_device_private_entry( 3037 + page_to_pfn(page + i)); 3038 + else 3039 + swp_entry = make_readable_device_private_entry( 3040 + page_to_pfn(page + i)); 3041 + /* 3042 + * Young and dirty bits are not progated via swp_entry 3043 + */ 3044 + entry = swp_entry_to_pte(swp_entry); 3045 + if (soft_dirty) 3046 + entry = pte_swp_mksoft_dirty(entry); 3047 + if (uffd_wp) 3048 + entry = pte_swp_mkuffd_wp(entry); 3056 3049 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3057 3050 set_pte_at(mm, addr, pte + i, entry); 3058 3051 } ··· 3105 3046 } 3106 3047 pte_unmap(pte); 3107 3048 3108 - if (!pmd_migration) 3049 + if (!is_pmd_migration_entry(*pmd)) 3109 3050 folio_remove_rmap_pmd(folio, page, vma); 3110 3051 if (freeze) 3111 3052 put_page(page); ··· 3118 3059 pmd_t *pmd, bool freeze) 3119 3060 { 3120 3061 VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); 3121 - if (pmd_trans_huge(*pmd) || is_pmd_migration_entry(*pmd)) 3062 + if (pmd_trans_huge(*pmd) || is_pmd_non_present_folio_entry(*pmd)) 3122 3063 __split_huge_pmd_locked(vma, pmd, address, freeze); 3123 3064 } 3124 3065 ··· 3296 3237 { 3297 3238 VM_BUG_ON_FOLIO(folio_test_lru(new_folio), folio); 3298 3239 lockdep_assert_held(&lruvec->lru_lock); 3240 + 3241 + if (folio_is_device_private(folio)) 3242 + return; 3299 3243 3300 3244 if (list) { 3301 3245 /* page reclaim is reclaiming a huge page */ ··· 3904 3842 if (nr_shmem_dropped) 3905 3843 shmem_uncharge(mapping->host, nr_shmem_dropped); 3906 3844 3907 - if (!ret && is_anon) 3845 + if (!ret && is_anon && !folio_is_device_private(folio)) 3908 3846 remap_flags = RMP_USE_SHARED_ZEROPAGE; 3847 + 3909 3848 remap_page(folio, 1 << order, remap_flags); 3910 3849 3911 3850 /*
+1
mm/migrate.c
··· 307 307 VM_BUG_ON_PAGE(!PageAnon(page), page); 308 308 VM_BUG_ON_PAGE(!PageLocked(page), page); 309 309 VM_BUG_ON_PAGE(pte_present(old_pte), page); 310 + VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio); 310 311 311 312 if (folio_test_mlocked(folio) || (pvmw->vma->vm_flags & VM_LOCKED) || 312 313 mm_forbids_zeropage(pvmw->vma->vm_mm))