mm: remove non_swap_entry() and use softleaf helpers instead

+10 -10

arch/s390/mm/gmap_helpers.c

··· 11 11 #include <linux/mm.h> 12 12 #include <linux/hugetlb.h> 13 13 #include <linux/swap.h> 14 - #include <linux/swapops.h> 14 + #include <linux/leafops.h> 15 15 #include <linux/pagewalk.h> 16 16 #include <linux/ksm.h> 17 17 #include <asm/gmap_helpers.h> 18 18 #include <asm/pgtable.h> 19 19 20 20 /** 21 - * ptep_zap_swap_entry() - discard a swap entry. 21 + * ptep_zap_softleaf_entry() - discard a software leaf entry. 22 22 * @mm: the mm 23 - * @entry: the swap entry that needs to be zapped 23 + * @entry: the software leaf entry that needs to be zapped 24 24 * 25 - * Discards the given swap entry. If the swap entry was an actual swap 26 - * entry (and not a migration entry, for example), the actual swapped 25 + * Discards the given software leaf entry. If the leaf entry was an actual 26 + * swap entry (and not a migration entry, for example), the actual swapped 27 27 * page is also discarded from swap. 28 28 */ 29 - static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) 29 + static void ptep_zap_softleaf_entry(struct mm_struct *mm, softleaf_t entry) 30 30 { 31 - if (!non_swap_entry(entry)) 31 + if (softleaf_is_swap(entry)) 32 32 dec_mm_counter(mm, MM_SWAPENTS); 33 - else if (is_migration_entry(entry)) 34 - dec_mm_counter(mm, mm_counter(pfn_swap_entry_folio(entry))); 33 + else if (softleaf_is_migration(entry)) 34 + dec_mm_counter(mm, mm_counter(softleaf_to_folio(entry))); 35 35 free_swap_and_cache(entry); 36 36 } 37 37 ··· 66 66 preempt_disable(); 67 67 pgste = pgste_get_lock(ptep); 68 68 69 - ptep_zap_swap_entry(mm, pte_to_swp_entry(*ptep)); 69 + ptep_zap_softleaf_entry(mm, softleaf_from_pte(*ptep)); 70 70 pte_clear(mm, vmaddr, ptep); 71 71 72 72 pgste_set_unlock(ptep, pgste);

+6 -6

arch/s390/mm/pgtable.c

··· 16 16 #include <linux/spinlock.h> 17 17 #include <linux/rcupdate.h> 18 18 #include <linux/slab.h> 19 - #include <linux/swapops.h> 19 + #include <linux/leafops.h> 20 20 #include <linux/sysctl.h> 21 21 #include <linux/ksm.h> 22 22 #include <linux/mman.h> ··· 683 683 pgste_set_unlock(ptep, pgste); 684 684 } 685 685 686 - static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) 686 + static void ptep_zap_softleaf_entry(struct mm_struct *mm, softleaf_t entry) 687 687 { 688 - if (!non_swap_entry(entry)) 688 + if (softleaf_is_swap(entry)) 689 689 dec_mm_counter(mm, MM_SWAPENTS); 690 - else if (is_migration_entry(entry)) { 691 - struct folio *folio = pfn_swap_entry_folio(entry); 690 + else if (softleaf_is_migration(entry)) { 691 + struct folio *folio = softleaf_to_folio(entry); 692 692 693 693 dec_mm_counter(mm, mm_counter(folio)); 694 694 } ··· 710 710 if (!reset && pte_swap(pte) && 711 711 ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || 712 712 (pgstev & _PGSTE_GPS_ZERO))) { 713 - ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); 713 + ptep_zap_softleaf_entry(mm, softleaf_from_pte(pte)); 714 714 pte_clear(mm, addr, ptep); 715 715 } 716 716 if (reset)

+6 -6

fs/proc/task_mmu.c

··· 1020 1020 } else if (pte_none(ptent)) { 1021 1021 smaps_pte_hole_lookup(addr, walk); 1022 1022 } else { 1023 - swp_entry_t swpent = pte_to_swp_entry(ptent); 1023 + const softleaf_t entry = softleaf_from_pte(ptent); 1024 1024 1025 - if (!non_swap_entry(swpent)) { 1025 + if (softleaf_is_swap(entry)) { 1026 1026 int mapcount; 1027 1027 1028 1028 mss->swap += PAGE_SIZE; 1029 - mapcount = swp_swapcount(swpent); 1029 + mapcount = swp_swapcount(entry); 1030 1030 if (mapcount >= 2) { 1031 1031 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; 1032 1032 ··· 1035 1035 } else { 1036 1036 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; 1037 1037 } 1038 - } else if (is_pfn_swap_entry(swpent)) { 1039 - if (is_device_private_entry(swpent)) 1038 + } else if (softleaf_has_pfn(entry)) { 1039 + if (softleaf_is_device_private(entry)) 1040 1040 present = true; 1041 - page = pfn_swap_entry_to_page(swpent); 1041 + page = softleaf_to_page(entry); 1042 1042 } 1043 1043 } 1044 1044

-5

include/linux/swapops.h

··· 492 492 493 493 #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ 494 494 495 - static inline int non_swap_entry(swp_entry_t entry) 496 - { 497 - return swp_type(entry) >= MAX_SWAPFILES; 498 - } 499 - 500 495 #endif /* CONFIG_MMU */ 501 496 #endif /* _LINUX_SWAPOPS_H */

+1 -1

mm/filemap.c

··· 4567 4567 swp_entry_t swp = radix_to_swp_entry(folio); 4568 4568 4569 4569 /* swapin error results in poisoned entry */ 4570 - if (non_swap_entry(swp)) 4570 + if (!softleaf_is_swap(swp)) 4571 4571 goto resched; 4572 4572 4573 4573 /*

+8 -8

mm/hmm.c

··· 258 258 } 259 259 260 260 if (!pte_present(pte)) { 261 - swp_entry_t entry = pte_to_swp_entry(pte); 261 + const softleaf_t entry = softleaf_from_pte(pte); 262 262 263 263 /* 264 264 * Don't fault in device private pages owned by the caller, 265 265 * just report the PFN. 266 266 */ 267 - if (is_device_private_entry(entry) && 268 - page_pgmap(pfn_swap_entry_to_page(entry))->owner == 267 + if (softleaf_is_device_private(entry) && 268 + page_pgmap(softleaf_to_page(entry))->owner == 269 269 range->dev_private_owner) { 270 270 cpu_flags = HMM_PFN_VALID; 271 - if (is_writable_device_private_entry(entry)) 271 + if (softleaf_is_device_private_write(entry)) 272 272 cpu_flags |= HMM_PFN_WRITE; 273 273 new_pfn_flags = swp_offset_pfn(entry) | cpu_flags; 274 274 goto out; ··· 279 279 if (!required_fault) 280 280 goto out; 281 281 282 - if (!non_swap_entry(entry)) 282 + if (softleaf_is_swap(entry)) 283 283 goto fault; 284 284 285 - if (is_device_private_entry(entry)) 285 + if (softleaf_is_device_private(entry)) 286 286 goto fault; 287 287 288 - if (is_device_exclusive_entry(entry)) 288 + if (softleaf_is_device_exclusive(entry)) 289 289 goto fault; 290 290 291 - if (is_migration_entry(entry)) { 291 + if (softleaf_is_migration(entry)) { 292 292 pte_unmap(ptep); 293 293 hmm_vma_walk->last = addr; 294 294 migration_entry_wait(walk->mm, pmdp, addr);

+1 -1

mm/madvise.c

··· 249 249 continue; 250 250 entry = radix_to_swp_entry(folio); 251 251 /* There might be swapin error entries in shmem mapping. */ 252 - if (non_swap_entry(entry)) 252 + if (!softleaf_is_swap(entry)) 253 253 continue; 254 254 255 255 addr = vma->vm_start +

+18 -18

mm/memory.c

··· 932 932 struct folio *folio; 933 933 struct page *page; 934 934 935 - if (likely(!non_swap_entry(entry))) { 935 + if (likely(softleaf_is_swap(entry))) { 936 936 if (swap_duplicate(entry) < 0) 937 937 return -EIO; 938 938 ··· 950 950 set_pte_at(src_mm, addr, src_pte, pte); 951 951 } 952 952 rss[MM_SWAPENTS]++; 953 - } else if (is_migration_entry(entry)) { 954 - folio = pfn_swap_entry_folio(entry); 953 + } else if (softleaf_is_migration(entry)) { 954 + folio = softleaf_to_folio(entry); 955 955 956 956 rss[mm_counter(folio)]++; 957 957 958 - if (!is_readable_migration_entry(entry) && 958 + if (!softleaf_is_migration_read(entry) && 959 959 is_cow_mapping(vm_flags)) { 960 960 /* 961 961 * COW mappings require pages in both parent and child ··· 964 964 */ 965 965 entry = make_readable_migration_entry( 966 966 swp_offset(entry)); 967 - pte = swp_entry_to_pte(entry); 967 + pte = softleaf_to_pte(entry); 968 968 if (pte_swp_soft_dirty(orig_pte)) 969 969 pte = pte_swp_mksoft_dirty(pte); 970 970 if (pte_swp_uffd_wp(orig_pte)) 971 971 pte = pte_swp_mkuffd_wp(pte); 972 972 set_pte_at(src_mm, addr, src_pte, pte); 973 973 } 974 - } else if (is_device_private_entry(entry)) { 975 - page = pfn_swap_entry_to_page(entry); 974 + } else if (softleaf_is_device_private(entry)) { 975 + page = softleaf_to_page(entry); 976 976 folio = page_folio(page); 977 977 978 978 /* ··· 996 996 * when a device driver is involved (you cannot easily 997 997 * save and restore device driver state). 998 998 */ 999 - if (is_writable_device_private_entry(entry) && 999 + if (softleaf_is_device_private_write(entry) && 1000 1000 is_cow_mapping(vm_flags)) { 1001 1001 entry = make_readable_device_private_entry( 1002 1002 swp_offset(entry)); ··· 1005 1005 pte = pte_swp_mkuffd_wp(pte); 1006 1006 set_pte_at(src_mm, addr, src_pte, pte); 1007 1007 } 1008 - } else if (is_device_exclusive_entry(entry)) { 1008 + } else if (softleaf_is_device_exclusive(entry)) { 1009 1009 /* 1010 1010 * Make device exclusive entries present by restoring the 1011 1011 * original entry then copying as for a present pte. Device ··· 4625 4625 rmap_t rmap_flags = RMAP_NONE; 4626 4626 bool need_clear_cache = false; 4627 4627 bool exclusive = false; 4628 - swp_entry_t entry; 4628 + softleaf_t entry; 4629 4629 pte_t pte; 4630 4630 vm_fault_t ret = 0; 4631 4631 void *shadow = NULL; ··· 4637 4637 if (!pte_unmap_same(vmf)) 4638 4638 goto out; 4639 4639 4640 - entry = pte_to_swp_entry(vmf->orig_pte); 4641 - if (unlikely(non_swap_entry(entry))) { 4642 - if (is_migration_entry(entry)) { 4640 + entry = softleaf_from_pte(vmf->orig_pte); 4641 + if (unlikely(!softleaf_is_swap(entry))) { 4642 + if (softleaf_is_migration(entry)) { 4643 4643 migration_entry_wait(vma->vm_mm, vmf->pmd, 4644 4644 vmf->address); 4645 - } else if (is_device_exclusive_entry(entry)) { 4646 - vmf->page = pfn_swap_entry_to_page(entry); 4645 + } else if (softleaf_is_device_exclusive(entry)) { 4646 + vmf->page = softleaf_to_page(entry); 4647 4647 ret = remove_device_exclusive_entry(vmf); 4648 - } else if (is_device_private_entry(entry)) { 4648 + } else if (softleaf_is_device_private(entry)) { 4649 4649 if (vmf->flags & FAULT_FLAG_VMA_LOCK) { 4650 4650 /* 4651 4651 * migrate_to_ram is not yet ready to operate ··· 4656 4656 goto out; 4657 4657 } 4658 4658 4659 - vmf->page = pfn_swap_entry_to_page(entry); 4659 + vmf->page = softleaf_to_page(entry); 4660 4660 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, 4661 4661 vmf->address, &vmf->ptl); 4662 4662 if (unlikely(!vmf->pte || ··· 4680 4680 } else { 4681 4681 pte_unmap_unlock(vmf->pte, vmf->ptl); 4682 4682 } 4683 - } else if (is_hwpoison_entry(entry)) { 4683 + } else if (softleaf_is_hwpoison(entry)) { 4684 4684 ret = VM_FAULT_HWPOISON; 4685 4685 } else if (softleaf_is_marker(entry)) { 4686 4686 ret = handle_pte_marker(vmf);

+1 -1

mm/mincore.c

··· 74 74 * absent. Page table may contain migration or hwpoison 75 75 * entries which are always uptodate. 76 76 */ 77 - if (non_swap_entry(entry)) 77 + if (!softleaf_is_swap(entry)) 78 78 return !shmem; 79 79 80 80 /*

+12 -12

mm/userfaultfd.c

··· 1256 1256 unsigned long dst_addr, unsigned long src_addr, 1257 1257 unsigned long len, __u64 mode) 1258 1258 { 1259 - swp_entry_t entry; 1260 1259 struct swap_info_struct *si = NULL; 1261 1260 pte_t orig_src_pte, orig_dst_pte; 1262 1261 pte_t src_folio_pte; ··· 1429 1430 orig_dst_pte, orig_src_pte, dst_pmd, 1430 1431 dst_pmdval, dst_ptl, src_ptl, &src_folio, 1431 1432 len); 1432 - } else { 1433 + } else { /* !pte_present() */ 1433 1434 struct folio *folio = NULL; 1435 + const softleaf_t entry = softleaf_from_pte(orig_src_pte); 1434 1436 1435 - entry = pte_to_swp_entry(orig_src_pte); 1436 - if (non_swap_entry(entry)) { 1437 - if (is_migration_entry(entry)) { 1438 - pte_unmap(src_pte); 1439 - pte_unmap(dst_pte); 1440 - src_pte = dst_pte = NULL; 1441 - migration_entry_wait(mm, src_pmd, src_addr); 1442 - ret = -EAGAIN; 1443 - } else 1444 - ret = -EFAULT; 1437 + if (softleaf_is_migration(entry)) { 1438 + pte_unmap(src_pte); 1439 + pte_unmap(dst_pte); 1440 + src_pte = dst_pte = NULL; 1441 + migration_entry_wait(mm, src_pmd, src_addr); 1442 + 1443 + ret = -EAGAIN; 1444 + goto out; 1445 + } else if (!softleaf_is_swap(entry)) { 1446 + ret = -EFAULT; 1445 1447 goto out; 1446 1448 } 1447 1449

Configure Feed

Configure Feed