fs/proc/task_mmu: refactor pagemap_pmd_range()

Separate out THP logic so we can drop an indentation level and reduce the
amount of noise in this function.

We add pagemap_pmd_range_thp() for this purpose.

While we're here, convert the VM_BUG_ON() to a VM_WARN_ON_ONCE() at the
same time.

No functional change intended.

Link: https://lkml.kernel.org/r/f9ce7f3bb57e3627288225e23f2498cc5315f5ab.1762812360.git.lorenzo.stoakes@oracle.com
Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Chris Li <chrisl@kernel.org>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Claudio Imbrenda <imbrenda@linux.ibm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Janosch Frank <frankja@linux.ibm.com>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Kairui Song <kasong@tencent.com>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Leon Romanovsky <leon@kernel.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Mathew Brost <matthew.brost@intel.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Naoya Horiguchi <nao.horiguchi@gmail.com>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Nico Pache <npache@redhat.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Wei Xu <weixugc@google.com>
Cc: xu xin <xu.xin16@zte.com.cn>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Lorenzo Stoakes and committed by

Andrew Morton 6 months ago de4d6c94 fb410d8b

+78 -70

1 changed file

expand all

proc

task_mmu.c

+78 -70

fs/proc/task_mmu.c

··· 1985 1985 return make_pme(frame, flags); 1986 1986 } 1987 1987 1988 + #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1989 + static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, 1990 + unsigned long end, struct vm_area_struct *vma, 1991 + struct pagemapread *pm) 1992 + { 1993 + unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; 1994 + u64 flags = 0, frame = 0; 1995 + pmd_t pmd = *pmdp; 1996 + struct page *page = NULL; 1997 + struct folio *folio = NULL; 1998 + int err = 0; 1999 + 2000 + if (vma->vm_flags & VM_SOFTDIRTY) 2001 + flags |= PM_SOFT_DIRTY; 2002 + 2003 + if (pmd_present(pmd)) { 2004 + page = pmd_page(pmd); 2005 + 2006 + flags |= PM_PRESENT; 2007 + if (pmd_soft_dirty(pmd)) 2008 + flags |= PM_SOFT_DIRTY; 2009 + if (pmd_uffd_wp(pmd)) 2010 + flags |= PM_UFFD_WP; 2011 + if (pm->show_pfn) 2012 + frame = pmd_pfn(pmd) + idx; 2013 + } else if (thp_migration_supported() && is_swap_pmd(pmd)) { 2014 + swp_entry_t entry = pmd_to_swp_entry(pmd); 2015 + unsigned long offset; 2016 + 2017 + if (pm->show_pfn) { 2018 + if (is_pfn_swap_entry(entry)) 2019 + offset = swp_offset_pfn(entry) + idx; 2020 + else 2021 + offset = swp_offset(entry) + idx; 2022 + frame = swp_type(entry) | 2023 + (offset << MAX_SWAPFILES_SHIFT); 2024 + } 2025 + flags |= PM_SWAP; 2026 + if (pmd_swp_soft_dirty(pmd)) 2027 + flags |= PM_SOFT_DIRTY; 2028 + if (pmd_swp_uffd_wp(pmd)) 2029 + flags |= PM_UFFD_WP; 2030 + VM_WARN_ON_ONCE(!is_pmd_migration_entry(pmd)); 2031 + page = pfn_swap_entry_to_page(entry); 2032 + } 2033 + 2034 + if (page) { 2035 + folio = page_folio(page); 2036 + if (!folio_test_anon(folio)) 2037 + flags |= PM_FILE; 2038 + } 2039 + 2040 + for (; addr != end; addr += PAGE_SIZE, idx++) { 2041 + u64 cur_flags = flags; 2042 + pagemap_entry_t pme; 2043 + 2044 + if (folio && (flags & PM_PRESENT) && 2045 + __folio_page_mapped_exclusively(folio, page)) 2046 + cur_flags |= PM_MMAP_EXCLUSIVE; 2047 + 2048 + pme = make_pme(frame, cur_flags); 2049 + err = add_to_pagemap(&pme, pm); 2050 + if (err) 2051 + break; 2052 + if (pm->show_pfn) { 2053 + if (flags & PM_PRESENT) 2054 + frame++; 2055 + else if (flags & PM_SWAP) 2056 + frame += (1 << MAX_SWAPFILES_SHIFT); 2057 + } 2058 + } 2059 + return err; 2060 + } 2061 + #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2062 + 1988 2063 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, 1989 2064 struct mm_walk *walk) 1990 2065 { ··· 2068 1993 spinlock_t *ptl; 2069 1994 pte_t *pte, *orig_pte; 2070 1995 int err = 0; 2071 - #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2072 1996 1997 + #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2073 1998 ptl = pmd_trans_huge_lock(pmdp, vma); 2074 1999 if (ptl) { 2075 - unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; 2076 - u64 flags = 0, frame = 0; 2077 - pmd_t pmd = *pmdp; 2078 - struct page *page = NULL; 2079 - struct folio *folio = NULL; 2080 - 2081 - if (vma->vm_flags & VM_SOFTDIRTY) 2082 - flags |= PM_SOFT_DIRTY; 2083 - 2084 - if (pmd_present(pmd)) { 2085 - page = pmd_page(pmd); 2086 - 2087 - flags |= PM_PRESENT; 2088 - if (pmd_soft_dirty(pmd)) 2089 - flags |= PM_SOFT_DIRTY; 2090 - if (pmd_uffd_wp(pmd)) 2091 - flags |= PM_UFFD_WP; 2092 - if (pm->show_pfn) 2093 - frame = pmd_pfn(pmd) + idx; 2094 - } 2095 - #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2096 - else if (is_swap_pmd(pmd)) { 2097 - swp_entry_t entry = pmd_to_swp_entry(pmd); 2098 - unsigned long offset; 2099 - 2100 - if (pm->show_pfn) { 2101 - if (is_pfn_swap_entry(entry)) 2102 - offset = swp_offset_pfn(entry) + idx; 2103 - else 2104 - offset = swp_offset(entry) + idx; 2105 - frame = swp_type(entry) | 2106 - (offset << MAX_SWAPFILES_SHIFT); 2107 - } 2108 - flags |= PM_SWAP; 2109 - if (pmd_swp_soft_dirty(pmd)) 2110 - flags |= PM_SOFT_DIRTY; 2111 - if (pmd_swp_uffd_wp(pmd)) 2112 - flags |= PM_UFFD_WP; 2113 - VM_BUG_ON(!is_pmd_migration_entry(pmd)); 2114 - page = pfn_swap_entry_to_page(entry); 2115 - } 2116 - #endif 2117 - 2118 - if (page) { 2119 - folio = page_folio(page); 2120 - if (!folio_test_anon(folio)) 2121 - flags |= PM_FILE; 2122 - } 2123 - 2124 - for (; addr != end; addr += PAGE_SIZE, idx++) { 2125 - u64 cur_flags = flags; 2126 - pagemap_entry_t pme; 2127 - 2128 - if (folio && (flags & PM_PRESENT) && 2129 - __folio_page_mapped_exclusively(folio, page)) 2130 - cur_flags |= PM_MMAP_EXCLUSIVE; 2131 - 2132 - pme = make_pme(frame, cur_flags); 2133 - err = add_to_pagemap(&pme, pm); 2134 - if (err) 2135 - break; 2136 - if (pm->show_pfn) { 2137 - if (flags & PM_PRESENT) 2138 - frame++; 2139 - else if (flags & PM_SWAP) 2140 - frame += (1 << MAX_SWAPFILES_SHIFT); 2141 - } 2142 - } 2000 + err = pagemap_pmd_range_thp(pmdp, addr, end, vma, pm); 2143 2001 spin_unlock(ptl); 2144 2002 return err; 2145 2003 } 2146 - #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2004 + #endif 2147 2005 2148 2006 /* 2149 2007 * We can assume that @vma always points to a valid one and @end never

Configure Feed

Configure Feed