tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm: remove is_hugetlb_entry_[migration, hwpoisoned]()

We do not need to have explicit helper functions for these, it adds a
level of confusion and indirection when we can simply use software leaf
entry logic here instead and spell out the special huge_pte_none() case we
must consider.

No functional change intended.

Link: https://lkml.kernel.org/r/0e92d6924d3de88cd014ce1c53e20edc08fc152e.1762812360.git.lorenzo.stoakes@oracle.com
Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Chris Li <chrisl@kernel.org>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Claudio Imbrenda <imbrenda@linux.ibm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Janosch Frank <frankja@linux.ibm.com>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Kairui Song <kasong@tencent.com>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Leon Romanovsky <leon@kernel.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Mathew Brost <matthew.brost@intel.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Naoya Horiguchi <nao.horiguchi@gmail.com>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Nico Pache <npache@redhat.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wei Xu <weixugc@google.com>
Cc: xu xin <xu.xin16@zte.com.cn>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Lorenzo Stoakes and committed by
Andrew Morton 03bfbc3a 9ff30bb9

+73 -79
+14 -13
fs/proc/task_mmu.c
··· 2500 2500 unsigned long addr, pte_t *ptep, 2501 2501 pte_t ptent) 2502 2502 { 2503 - unsigned long psize; 2503 + const unsigned long psize = huge_page_size(hstate_vma(vma)); 2504 + softleaf_t entry; 2504 2505 2505 - if (is_hugetlb_entry_hwpoisoned(ptent) || pte_is_marker(ptent)) 2506 - return; 2507 - 2508 - psize = huge_page_size(hstate_vma(vma)); 2509 - 2510 - if (is_hugetlb_entry_migration(ptent)) 2511 - set_huge_pte_at(vma->vm_mm, addr, ptep, 2512 - pte_swp_mkuffd_wp(ptent), psize); 2513 - else if (!huge_pte_none(ptent)) 2514 - huge_ptep_modify_prot_commit(vma, addr, ptep, ptent, 2515 - huge_pte_mkuffd_wp(ptent)); 2516 - else 2506 + if (huge_pte_none(ptent)) 2517 2507 set_huge_pte_at(vma->vm_mm, addr, ptep, 2518 2508 make_pte_marker(PTE_MARKER_UFFD_WP), psize); 2509 + 2510 + entry = softleaf_from_pte(ptent); 2511 + if (softleaf_is_hwpoison(entry) || softleaf_is_marker(entry)) 2512 + return; 2513 + 2514 + if (softleaf_is_migration(entry)) 2515 + set_huge_pte_at(vma->vm_mm, addr, ptep, 2516 + pte_swp_mkuffd_wp(ptent), psize); 2517 + else 2518 + huge_ptep_modify_prot_commit(vma, addr, ptep, ptent, 2519 + huge_pte_mkuffd_wp(ptent)); 2519 2520 } 2520 2521 #endif /* CONFIG_HUGETLB_PAGE */ 2521 2522
-2
include/linux/hugetlb.h
··· 274 274 long hugetlb_change_protection(struct vm_area_struct *vma, 275 275 unsigned long address, unsigned long end, pgprot_t newprot, 276 276 unsigned long cp_flags); 277 - bool is_hugetlb_entry_migration(pte_t pte); 278 - bool is_hugetlb_entry_hwpoisoned(pte_t pte); 279 277 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma); 280 278 void fixup_hugetlb_reservations(struct vm_area_struct *vma); 281 279 void hugetlb_split(struct vm_area_struct *vma, unsigned long addr);
+37 -54
mm/hugetlb.c
··· 4846 4846 set_huge_ptep_writable(vma, address, ptep); 4847 4847 } 4848 4848 4849 - bool is_hugetlb_entry_migration(pte_t pte) 4850 - { 4851 - swp_entry_t swp; 4852 - 4853 - if (huge_pte_none(pte) || pte_present(pte)) 4854 - return false; 4855 - swp = pte_to_swp_entry(pte); 4856 - if (is_migration_entry(swp)) 4857 - return true; 4858 - else 4859 - return false; 4860 - } 4861 - 4862 - bool is_hugetlb_entry_hwpoisoned(pte_t pte) 4863 - { 4864 - swp_entry_t swp; 4865 - 4866 - if (huge_pte_none(pte) || pte_present(pte)) 4867 - return false; 4868 - swp = pte_to_swp_entry(pte); 4869 - if (is_hwpoison_entry(swp)) 4870 - return true; 4871 - else 4872 - return false; 4873 - } 4874 - 4875 4849 static void 4876 4850 hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr, 4877 4851 struct folio *new_folio, pte_t old, unsigned long sz) ··· 4874 4900 unsigned long npages = pages_per_huge_page(h); 4875 4901 struct mmu_notifier_range range; 4876 4902 unsigned long last_addr_mask; 4903 + softleaf_t softleaf; 4877 4904 int ret = 0; 4878 4905 4879 4906 if (cow) { ··· 4922 4947 entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte); 4923 4948 again: 4924 4949 if (huge_pte_none(entry)) { 4925 - /* 4926 - * Skip if src entry none. 4927 - */ 4928 - ; 4929 - } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) { 4950 + /* Skip if src entry none. */ 4951 + goto next; 4952 + } 4953 + 4954 + softleaf = softleaf_from_pte(entry); 4955 + if (unlikely(softleaf_is_hwpoison(softleaf))) { 4930 4956 if (!userfaultfd_wp(dst_vma)) 4931 4957 entry = huge_pte_clear_uffd_wp(entry); 4932 4958 set_huge_pte_at(dst, addr, dst_pte, entry, sz); 4933 - } else if (unlikely(is_hugetlb_entry_migration(entry))) { 4934 - softleaf_t softleaf = softleaf_from_pte(entry); 4959 + } else if (unlikely(softleaf_is_migration(softleaf))) { 4935 4960 bool uffd_wp = pte_swp_uffd_wp(entry); 4936 4961 4937 4962 if (!is_readable_migration_entry(softleaf) && cow) { ··· 4950 4975 entry = huge_pte_clear_uffd_wp(entry); 4951 4976 set_huge_pte_at(dst, addr, dst_pte, entry, sz); 4952 4977 } else if (unlikely(pte_is_marker(entry))) { 4953 - const softleaf_t softleaf = softleaf_from_pte(entry); 4954 4978 const pte_marker marker = copy_pte_marker(softleaf, dst_vma); 4955 4979 4956 4980 if (marker) ··· 5007 5033 } 5008 5034 hugetlb_install_folio(dst_vma, dst_pte, addr, 5009 5035 new_folio, src_pte_old, sz); 5010 - spin_unlock(src_ptl); 5011 - spin_unlock(dst_ptl); 5012 - continue; 5036 + goto next; 5013 5037 } 5014 5038 5015 5039 if (cow) { ··· 5028 5056 set_huge_pte_at(dst, addr, dst_pte, entry, sz); 5029 5057 hugetlb_count_add(npages, dst); 5030 5058 } 5059 + 5060 + next: 5031 5061 spin_unlock(src_ptl); 5032 5062 spin_unlock(dst_ptl); 5033 5063 } ··· 6038 6064 ret = 0; 6039 6065 6040 6066 /* Not present, either a migration or a hwpoisoned entry */ 6041 - if (!pte_present(vmf.orig_pte)) { 6042 - if (is_hugetlb_entry_migration(vmf.orig_pte)) { 6067 + if (!pte_present(vmf.orig_pte) && !huge_pte_none(vmf.orig_pte)) { 6068 + const softleaf_t softleaf = softleaf_from_pte(vmf.orig_pte); 6069 + 6070 + if (softleaf_is_migration(softleaf)) { 6043 6071 /* 6044 6072 * Release the hugetlb fault lock now, but retain 6045 6073 * the vma lock, because it is needed to guard the ··· 6052 6076 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 6053 6077 migration_entry_wait_huge(vma, vmf.address, vmf.pte); 6054 6078 return 0; 6055 - } else if (is_hugetlb_entry_hwpoisoned(vmf.orig_pte)) 6079 + } 6080 + if (softleaf_is_hwpoison(softleaf)) { 6056 6081 ret = VM_FAULT_HWPOISON_LARGE | 6057 6082 VM_FAULT_SET_HINDEX(hstate_index(h)); 6083 + } 6084 + 6058 6085 goto out_mutex; 6059 6086 } 6060 6087 ··· 6439 6460 i_mmap_lock_write(vma->vm_file->f_mapping); 6440 6461 last_addr_mask = hugetlb_mask_last_page(h); 6441 6462 for (; address < end; address += psize) { 6463 + softleaf_t entry; 6442 6464 spinlock_t *ptl; 6465 + 6443 6466 ptep = hugetlb_walk(vma, address, psize); 6444 6467 if (!ptep) { 6445 6468 if (!uffd_wp) { ··· 6473 6492 continue; 6474 6493 } 6475 6494 pte = huge_ptep_get(mm, address, ptep); 6476 - if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 6477 - /* Nothing to do. */ 6478 - } else if (unlikely(is_hugetlb_entry_migration(pte))) { 6479 - softleaf_t entry = softleaf_from_pte(pte); 6495 + if (huge_pte_none(pte)) { 6496 + if (unlikely(uffd_wp)) 6497 + /* Safe to modify directly (none->non-present). */ 6498 + set_huge_pte_at(mm, address, ptep, 6499 + make_pte_marker(PTE_MARKER_UFFD_WP), 6500 + psize); 6501 + goto next; 6502 + } 6480 6503 6504 + entry = softleaf_from_pte(pte); 6505 + if (unlikely(softleaf_is_hwpoison(entry))) { 6506 + /* Nothing to do. */ 6507 + } else if (unlikely(softleaf_is_migration(entry))) { 6481 6508 struct folio *folio = softleaf_to_folio(entry); 6482 6509 pte_t newpte = pte; 6483 6510 6484 - if (is_writable_migration_entry(entry)) { 6511 + if (softleaf_is_migration_write(entry)) { 6485 6512 if (folio_test_anon(folio)) 6486 6513 entry = make_readable_exclusive_migration_entry( 6487 6514 swp_offset(entry)); ··· 6516 6527 if (pte_is_uffd_wp_marker(pte) && uffd_wp_resolve) 6517 6528 /* Safe to modify directly (non-present->none). */ 6518 6529 huge_pte_clear(mm, address, ptep, psize); 6519 - } else if (!huge_pte_none(pte)) { 6530 + } else { 6520 6531 pte_t old_pte; 6521 6532 unsigned int shift = huge_page_shift(hstate_vma(vma)); 6522 6533 ··· 6529 6540 pte = huge_pte_clear_uffd_wp(pte); 6530 6541 huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); 6531 6542 pages++; 6532 - } else { 6533 - /* None pte */ 6534 - if (unlikely(uffd_wp)) 6535 - /* Safe to modify directly (none->non-present). */ 6536 - set_huge_pte_at(mm, address, ptep, 6537 - make_pte_marker(PTE_MARKER_UFFD_WP), 6538 - psize); 6539 6543 } 6540 - spin_unlock(ptl); 6541 6544 6545 + next: 6546 + spin_unlock(ptl); 6542 6547 cond_resched(); 6543 6548 } 6544 6549 /*
+11 -6
mm/mempolicy.c
··· 768 768 unsigned long flags = qp->flags; 769 769 struct folio *folio; 770 770 spinlock_t *ptl; 771 - pte_t entry; 771 + pte_t ptep; 772 772 773 773 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); 774 - entry = huge_ptep_get(walk->mm, addr, pte); 775 - if (!pte_present(entry)) { 776 - if (unlikely(is_hugetlb_entry_migration(entry))) 777 - qp->nr_failed++; 774 + ptep = huge_ptep_get(walk->mm, addr, pte); 775 + if (!pte_present(ptep)) { 776 + if (!huge_pte_none(ptep)) { 777 + const softleaf_t entry = softleaf_from_pte(ptep); 778 + 779 + if (unlikely(softleaf_is_migration(entry))) 780 + qp->nr_failed++; 781 + } 782 + 778 783 goto unlock; 779 784 } 780 - folio = pfn_folio(pte_pfn(entry)); 785 + folio = pfn_folio(pte_pfn(ptep)); 781 786 if (!queue_folio_required(folio, qp)) 782 787 goto unlock; 783 788 if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
+11 -4
mm/migrate.c
··· 515 515 void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) 516 516 { 517 517 spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), vma->vm_mm, ptep); 518 + softleaf_t entry; 518 519 pte_t pte; 519 520 520 521 hugetlb_vma_assert_locked(vma); 521 522 spin_lock(ptl); 522 523 pte = huge_ptep_get(vma->vm_mm, addr, ptep); 523 524 524 - if (unlikely(!is_hugetlb_entry_migration(pte))) { 525 - spin_unlock(ptl); 526 - hugetlb_vma_unlock_read(vma); 527 - } else { 525 + if (huge_pte_none(pte)) 526 + goto fail; 527 + 528 + entry = softleaf_from_pte(pte); 529 + if (softleaf_is_migration(entry)) { 528 530 /* 529 531 * If migration entry existed, safe to release vma lock 530 532 * here because the pgtable page won't be freed without the ··· 535 533 */ 536 534 hugetlb_vma_unlock_read(vma); 537 535 migration_entry_wait_on_locked(pte_to_swp_entry(pte), ptl); 536 + return; 538 537 } 538 + 539 + fail: 540 + spin_unlock(ptl); 541 + hugetlb_vma_unlock_read(vma); 539 542 } 540 543 #endif 541 544