+2 -2

Documentation/admin-guide/mm/transhuge.rst

reviewed

··· 467 467 instead falls back to using huge pages with lower orders or 468 468 small pages even though the allocation was successful. 469 469 470 470 - anon_swpout 470 470 + swpout 471 471 is incremented every time a huge page is swapped out in one 472 472 piece without splitting. 473 473 474 474 - anon_swpout_fallback 474 474 + swpout_fallback 475 475 is incremented if a huge page has to be split before swapout. 476 476 Usually because failed to allocate some continuous swap space 477 477 for the huge page.

+2 -2

arch/arm64/mm/contpte.c

reviewed

··· 376 376 * clearing access/dirty for the whole block. 377 377 */ 378 378 unsigned long start = addr; 379 379 - unsigned long end = start + nr; 379 379 + unsigned long end = start + nr * PAGE_SIZE; 380 380 381 381 if (pte_cont(__ptep_get(ptep + nr - 1))) 382 382 end = ALIGN(end, CONT_PTE_SIZE); ··· 386 386 ptep = contpte_align_down(ptep); 387 387 } 388 388 389 389 - __clear_young_dirty_ptes(vma, start, ptep, end - start, flags); 389 389 + __clear_young_dirty_ptes(vma, start, ptep, (end - start) / PAGE_SIZE, flags); 390 390 } 391 391 EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); 392 392

+1 -1

fs/nilfs2/dir.c

reviewed

··· 607 607 608 608 kaddr = nilfs_get_folio(inode, i, &folio); 609 609 if (IS_ERR(kaddr)) 610 610 - continue; 610 610 + return 0; 611 611 612 612 de = (struct nilfs_dir_entry *)kaddr; 613 613 kaddr += nilfs_last_byte(inode, i) - NILFS_DIR_REC_LEN(1);

+3

fs/nilfs2/segment.c

reviewed

··· 1652 1652 if (bh->b_folio != bd_folio) { 1653 1653 if (bd_folio) { 1654 1654 folio_lock(bd_folio); 1655 1655 + folio_wait_writeback(bd_folio); 1655 1656 folio_clear_dirty_for_io(bd_folio); 1656 1657 folio_start_writeback(bd_folio); 1657 1658 folio_unlock(bd_folio); ··· 1666 1665 if (bh == segbuf->sb_super_root) { 1667 1666 if (bh->b_folio != bd_folio) { 1668 1667 folio_lock(bd_folio); 1668 1668 + folio_wait_writeback(bd_folio); 1669 1669 folio_clear_dirty_for_io(bd_folio); 1670 1670 folio_start_writeback(bd_folio); 1671 1671 folio_unlock(bd_folio); ··· 1683 1681 } 1684 1682 if (bd_folio) { 1685 1683 folio_lock(bd_folio); 1684 1684 + folio_wait_writeback(bd_folio); 1686 1685 folio_clear_dirty_for_io(bd_folio); 1687 1686 folio_start_writeback(bd_folio); 1688 1687 folio_unlock(bd_folio);

+1 -1

fs/proc/base.c

reviewed

··· 3214 3214 mm = get_task_mm(task); 3215 3215 if (mm) { 3216 3216 seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); 3217 3217 - seq_printf(m, "ksm_zero_pages %lu\n", mm->ksm_zero_pages); 3217 3217 + seq_printf(m, "ksm_zero_pages %ld\n", mm_ksm_zero_pages(mm)); 3218 3218 seq_printf(m, "ksm_merging_pages %lu\n", mm->ksm_merging_pages); 3219 3219 seq_printf(m, "ksm_process_profit %ld\n", ksm_process_profit(mm)); 3220 3220 mmput(mm);

+8 -2

include/linux/huge_mm.h

reviewed

··· 269 269 MTHP_STAT_ANON_FAULT_ALLOC, 270 270 MTHP_STAT_ANON_FAULT_FALLBACK, 271 271 MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, 272 272 - MTHP_STAT_ANON_SWPOUT, 273 273 - MTHP_STAT_ANON_SWPOUT_FALLBACK, 272 272 + MTHP_STAT_SWPOUT, 273 273 + MTHP_STAT_SWPOUT_FALLBACK, 274 274 __MTHP_STAT_COUNT 275 275 }; 276 276 ··· 278 278 unsigned long stats[ilog2(MAX_PTRS_PER_PTE) + 1][__MTHP_STAT_COUNT]; 279 279 }; 280 280 281 281 + #ifdef CONFIG_SYSFS 281 282 DECLARE_PER_CPU(struct mthp_stat, mthp_stats); 282 283 283 284 static inline void count_mthp_stat(int order, enum mthp_stat_item item) ··· 288 287 289 288 this_cpu_inc(mthp_stats.stats[order][item]); 290 289 } 290 290 + #else 291 291 + static inline void count_mthp_stat(int order, enum mthp_stat_item item) 292 292 + { 293 293 + } 294 294 + #endif 291 295 292 296 #define transparent_hugepage_use_zero_page() \ 293 297 (transparent_hugepage_flags & \

+14 -3

include/linux/ksm.h

reviewed

··· 33 33 */ 34 34 #define is_ksm_zero_pte(pte) (is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte)) 35 35 36 36 - extern unsigned long ksm_zero_pages; 36 36 + extern atomic_long_t ksm_zero_pages; 37 37 + 38 38 + static inline void ksm_map_zero_page(struct mm_struct *mm) 39 39 + { 40 40 + atomic_long_inc(&ksm_zero_pages); 41 41 + atomic_long_inc(&mm->ksm_zero_pages); 42 42 + } 37 43 38 44 static inline void ksm_might_unmap_zero_page(struct mm_struct *mm, pte_t pte) 39 45 { 40 46 if (is_ksm_zero_pte(pte)) { 41 41 - ksm_zero_pages--; 42 42 - mm->ksm_zero_pages--; 47 47 + atomic_long_dec(&ksm_zero_pages); 48 48 + atomic_long_dec(&mm->ksm_zero_pages); 43 49 } 50 50 + } 51 51 + 52 52 + static inline long mm_ksm_zero_pages(struct mm_struct *mm) 53 53 + { 54 54 + return atomic_long_read(&mm->ksm_zero_pages); 44 55 } 45 56 46 57 static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)

+1 -1

include/linux/mm_types.h

reviewed

··· 985 985 * Represent how many empty pages are merged with kernel zero 986 986 * pages when enabling KSM use_zero_pages. 987 987 */ 988 988 - unsigned long ksm_zero_pages; 988 988 + atomic_long_t ksm_zero_pages; 989 989 #endif /* CONFIG_KSM */ 990 990 #ifdef CONFIG_LRU_GEN_WALKS_MMU 991 991 struct {

+1 -1

mm/filemap.c

reviewed

··· 1000 1000 do { 1001 1001 cpuset_mems_cookie = read_mems_allowed_begin(); 1002 1002 n = cpuset_mem_spread_node(); 1003 1003 - folio = __folio_alloc_node(gfp, order, n); 1003 1003 + folio = __folio_alloc_node_noprof(gfp, order, n); 1004 1004 } while (!folio && read_mems_allowed_retry(cpuset_mems_cookie)); 1005 1005 1006 1006 return folio;

+4 -4

mm/huge_memory.c

reviewed

··· 558 558 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); 559 559 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); 560 560 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 561 561 - DEFINE_MTHP_STAT_ATTR(anon_swpout, MTHP_STAT_ANON_SWPOUT); 562 562 - DEFINE_MTHP_STAT_ATTR(anon_swpout_fallback, MTHP_STAT_ANON_SWPOUT_FALLBACK); 561 561 + DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); 562 562 + DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); 563 563 564 564 static struct attribute *stats_attrs[] = { 565 565 &anon_fault_alloc_attr.attr, 566 566 &anon_fault_fallback_attr.attr, 567 567 &anon_fault_fallback_charge_attr.attr, 568 568 - &anon_swpout_attr.attr, 569 569 - &anon_swpout_fallback_attr.attr, 568 568 + &swpout_attr.attr, 569 569 + &swpout_fallback_attr.attr, 570 570 NULL, 571 571 }; 572 572

+14 -2

mm/hugetlb.c

reviewed

··· 5768 5768 * do_exit() will not see it, and will keep the reservation 5769 5769 * forever. 5770 5770 */ 5771 5771 - if (adjust_reservation && vma_needs_reservation(h, vma, address)) 5772 5772 - vma_add_reservation(h, vma, address); 5771 5771 + if (adjust_reservation) { 5772 5772 + int rc = vma_needs_reservation(h, vma, address); 5773 5773 + 5774 5774 + if (rc < 0) 5775 5775 + /* Pressumably allocate_file_region_entries failed 5776 5776 + * to allocate a file_region struct. Clear 5777 5777 + * hugetlb_restore_reserve so that global reserve 5778 5778 + * count will not be incremented by free_huge_folio. 5779 5779 + * Act as if we consumed the reservation. 5780 5780 + */ 5781 5781 + folio_clear_hugetlb_restore_reserve(page_folio(page)); 5782 5782 + else if (rc) 5783 5783 + vma_add_reservation(h, vma, address); 5784 5784 + } 5773 5785 5774 5786 tlb_remove_page_size(tlb, page, huge_page_size(h)); 5775 5787 /*

+11 -4

mm/kmsan/core.c

reviewed

··· 196 196 u32 origin, bool checked) 197 197 { 198 198 u64 address = (u64)addr; 199 199 - void *shadow_start; 200 200 - u32 *origin_start; 199 199 + u32 *shadow_start, *origin_start; 201 200 size_t pad = 0; 202 201 203 202 KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(addr, size)); ··· 224 225 origin_start = 225 226 (u32 *)kmsan_get_metadata((void *)address, KMSAN_META_ORIGIN); 226 227 227 227 - for (int i = 0; i < size / KMSAN_ORIGIN_SIZE; i++) 228 228 - origin_start[i] = origin; 228 228 + /* 229 229 + * If the new origin is non-zero, assume that the shadow byte is also non-zero, 230 230 + * and unconditionally overwrite the old origin slot. 231 231 + * If the new origin is zero, overwrite the old origin slot iff the 232 232 + * corresponding shadow slot is zero. 233 233 + */ 234 234 + for (int i = 0; i < size / KMSAN_ORIGIN_SIZE; i++) { 235 235 + if (origin || !shadow_start[i]) 236 236 + origin_start[i] = origin; 237 237 + } 229 238 } 230 239 231 240 struct page *kmsan_vmalloc_to_page_or_null(void *vaddr)

+7 -10

mm/ksm.c

reviewed

··· 296 296 static bool ksm_smart_scan = true; 297 297 298 298 /* The number of zero pages which is placed by KSM */ 299 299 - unsigned long ksm_zero_pages; 299 299 + atomic_long_t ksm_zero_pages = ATOMIC_LONG_INIT(0); 300 300 301 301 /* The number of pages that have been skipped due to "smart scanning" */ 302 302 static unsigned long ksm_pages_skipped; ··· 1429 1429 * the dirty bit in zero page's PTE is set. 1430 1430 */ 1431 1431 newpte = pte_mkdirty(pte_mkspecial(pfn_pte(page_to_pfn(kpage), vma->vm_page_prot))); 1432 1432 - ksm_zero_pages++; 1433 1433 - mm->ksm_zero_pages++; 1432 1432 + ksm_map_zero_page(mm); 1434 1433 /* 1435 1434 * We're replacing an anonymous page with a zero page, which is 1436 1435 * not anonymous. We need to do proper accounting otherwise we ··· 2753 2754 { 2754 2755 struct ksm_rmap_item *rmap_item; 2755 2756 struct page *page; 2756 2756 - unsigned int npages = scan_npages; 2757 2757 2758 2758 - while (npages-- && likely(!freezing(current))) { 2758 2758 + while (scan_npages-- && likely(!freezing(current))) { 2759 2759 cond_resched(); 2760 2760 rmap_item = scan_get_next_rmap_item(&page); 2761 2761 if (!rmap_item) 2762 2762 return; 2763 2763 cmp_and_merge_page(page, rmap_item); 2764 2764 put_page(page); 2765 2765 + ksm_pages_scanned++; 2765 2766 } 2766 2766 - 2767 2767 - ksm_pages_scanned += scan_npages - npages; 2768 2767 } 2769 2768 2770 2769 static int ksmd_should_run(void) ··· 3373 3376 #ifdef CONFIG_PROC_FS 3374 3377 long ksm_process_profit(struct mm_struct *mm) 3375 3378 { 3376 3376 - return (long)(mm->ksm_merging_pages + mm->ksm_zero_pages) * PAGE_SIZE - 3379 3379 + return (long)(mm->ksm_merging_pages + mm_ksm_zero_pages(mm)) * PAGE_SIZE - 3377 3380 mm->ksm_rmap_items * sizeof(struct ksm_rmap_item); 3378 3381 } 3379 3382 #endif /* CONFIG_PROC_FS */ ··· 3662 3665 static ssize_t ksm_zero_pages_show(struct kobject *kobj, 3663 3666 struct kobj_attribute *attr, char *buf) 3664 3667 { 3665 3665 - return sysfs_emit(buf, "%ld\n", ksm_zero_pages); 3668 3668 + return sysfs_emit(buf, "%ld\n", atomic_long_read(&ksm_zero_pages)); 3666 3669 } 3667 3670 KSM_ATTR_RO(ksm_zero_pages); 3668 3671 ··· 3671 3674 { 3672 3675 long general_profit; 3673 3676 3674 3674 - general_profit = (ksm_pages_sharing + ksm_zero_pages) * PAGE_SIZE - 3677 3677 + general_profit = (ksm_pages_sharing + atomic_long_read(&ksm_zero_pages)) * PAGE_SIZE - 3675 3678 ksm_rmap_items * sizeof(struct ksm_rmap_item); 3676 3679 3677 3680 return sysfs_emit(buf, "%ld\n", general_profit);

-2

mm/memcontrol.c

reviewed

··· 3147 3147 struct mem_cgroup *memcg; 3148 3148 struct lruvec *lruvec; 3149 3149 3150 3150 - lockdep_assert_irqs_disabled(); 3151 3151 - 3152 3150 rcu_read_lock(); 3153 3151 memcg = obj_cgroup_memcg(objcg); 3154 3152 lruvec = mem_cgroup_lruvec(memcg, pgdat);

+1 -1

mm/mempool.c

reviewed

··· 273 273 { 274 274 mempool_t *pool; 275 275 276 276 - pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id); 276 276 + pool = kmalloc_node_noprof(sizeof(*pool), gfp_mask | __GFP_ZERO, node_id); 277 277 if (!pool) 278 278 return NULL; 279 279

+34 -16

mm/page_alloc.c

reviewed

··· 1955 1955 } 1956 1956 1957 1957 /* 1958 1958 - * Reserve a pageblock for exclusive use of high-order atomic allocations if 1959 1959 - * there are no empty page blocks that contain a page with a suitable order 1958 1958 + * Reserve the pageblock(s) surrounding an allocation request for 1959 1959 + * exclusive use of high-order atomic allocations if there are no 1960 1960 + * empty page blocks that contain a page with a suitable order 1960 1961 */ 1961 1961 - static void reserve_highatomic_pageblock(struct page *page, struct zone *zone) 1962 1962 + static void reserve_highatomic_pageblock(struct page *page, int order, 1963 1963 + struct zone *zone) 1962 1964 { 1963 1965 int mt; 1964 1966 unsigned long max_managed, flags; ··· 1986 1984 /* Yoink! */ 1987 1985 mt = get_pageblock_migratetype(page); 1988 1986 /* Only reserve normal pageblocks (i.e., they can merge with others) */ 1989 1989 - if (migratetype_is_mergeable(mt)) 1990 1990 - if (move_freepages_block(zone, page, mt, 1991 1991 - MIGRATE_HIGHATOMIC) != -1) 1992 1992 - zone->nr_reserved_highatomic += pageblock_nr_pages; 1987 1987 + if (!migratetype_is_mergeable(mt)) 1988 1988 + goto out_unlock; 1989 1989 + 1990 1990 + if (order < pageblock_order) { 1991 1991 + if (move_freepages_block(zone, page, mt, MIGRATE_HIGHATOMIC) == -1) 1992 1992 + goto out_unlock; 1993 1993 + zone->nr_reserved_highatomic += pageblock_nr_pages; 1994 1994 + } else { 1995 1995 + change_pageblock_range(page, order, MIGRATE_HIGHATOMIC); 1996 1996 + zone->nr_reserved_highatomic += 1 << order; 1997 1997 + } 1993 1998 1994 1999 out_unlock: 1995 2000 spin_unlock_irqrestore(&zone->lock, flags); ··· 2008 1999 * intense memory pressure but failed atomic allocations should be easier 2009 2000 * to recover from than an OOM. 2010 2001 * 2011 2011 - * If @force is true, try to unreserve a pageblock even though highatomic 2002 2002 + * If @force is true, try to unreserve pageblocks even though highatomic 2012 2003 * pageblock is exhausted. 2013 2004 */ 2014 2005 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, ··· 2050 2041 * adjust the count once. 2051 2042 */ 2052 2043 if (is_migrate_highatomic(mt)) { 2044 2044 + unsigned long size; 2053 2045 /* 2054 2046 * It should never happen but changes to 2055 2047 * locking could inadvertently allow a per-cpu ··· 2058 2048 * while unreserving so be safe and watch for 2059 2049 * underflows. 2060 2050 */ 2061 2061 - zone->nr_reserved_highatomic -= min( 2062 2062 - pageblock_nr_pages, 2063 2063 - zone->nr_reserved_highatomic); 2051 2051 + size = max(pageblock_nr_pages, 1UL << order); 2052 2052 + size = min(size, zone->nr_reserved_highatomic); 2053 2053 + zone->nr_reserved_highatomic -= size; 2064 2054 } 2065 2055 2066 2056 /* ··· 2072 2062 * of pageblocks that cannot be completely freed 2073 2063 * may increase. 2074 2064 */ 2075 2075 - ret = move_freepages_block(zone, page, mt, 2076 2076 - ac->migratetype); 2065 2065 + if (order < pageblock_order) 2066 2066 + ret = move_freepages_block(zone, page, mt, 2067 2067 + ac->migratetype); 2068 2068 + else { 2069 2069 + move_to_free_list(page, zone, order, mt, 2070 2070 + ac->migratetype); 2071 2071 + change_pageblock_range(page, order, 2072 2072 + ac->migratetype); 2073 2073 + ret = 1; 2074 2074 + } 2077 2075 /* 2078 2078 - * Reserving this block already succeeded, so this should 2079 2079 - * not fail on zone boundaries. 2076 2076 + * Reserving the block(s) already succeeded, 2077 2077 + * so this should not fail on zone boundaries. 2080 2078 */ 2081 2079 WARN_ON_ONCE(ret == -1); 2082 2080 if (ret > 0) { ··· 3424 3406 * if the pageblock should be reserved for the future 3425 3407 */ 3426 3408 if (unlikely(alloc_flags & ALLOC_HIGHATOMIC)) 3427 3427 - reserve_highatomic_pageblock(page, zone); 3409 3409 + reserve_highatomic_pageblock(page, order, zone); 3428 3410 3429 3411 return page; 3430 3412 } else {

+1 -1

mm/page_io.c

reviewed

··· 217 217 count_memcg_folio_events(folio, THP_SWPOUT, 1); 218 218 count_vm_event(THP_SWPOUT); 219 219 } 220 220 - count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_SWPOUT); 220 220 + count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT); 221 221 #endif 222 222 count_vm_events(PSWPOUT, folio_nr_pages(folio)); 223 223 }

+3 -2

mm/slub.c

reviewed

··· 1952 1952 #ifdef CONFIG_MEMCG 1953 1953 new_exts |= MEMCG_DATA_OBJEXTS; 1954 1954 #endif 1955 1955 - old_exts = slab->obj_exts; 1955 1955 + old_exts = READ_ONCE(slab->obj_exts); 1956 1956 handle_failed_objexts_alloc(old_exts, vec, objects); 1957 1957 if (new_slab) { 1958 1958 /* ··· 1961 1961 * be simply assigned. 1962 1962 */ 1963 1963 slab->obj_exts = new_exts; 1964 1964 - } else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) { 1964 1964 + } else if ((old_exts & ~OBJEXTS_FLAGS_MASK) || 1965 1965 + cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) { 1965 1966 /* 1966 1967 * If the slab is already in use, somebody can allocate and 1967 1968 * assign slabobj_exts in parallel. In this case the existing

+5 -5

mm/util.c

reviewed

··· 705 705 706 706 if (oldsize >= newsize) 707 707 return (void *)p; 708 708 - newp = kvmalloc(newsize, flags); 708 708 + newp = kvmalloc_noprof(newsize, flags); 709 709 if (!newp) 710 710 return NULL; 711 711 memcpy(newp, p, oldsize); ··· 726 726 727 727 if (unlikely(check_mul_overflow(n, size, &bytes))) 728 728 return NULL; 729 729 - return __vmalloc(bytes, flags); 729 729 + return __vmalloc_noprof(bytes, flags); 730 730 } 731 731 EXPORT_SYMBOL(__vmalloc_array_noprof); 732 732 ··· 737 737 */ 738 738 void *vmalloc_array_noprof(size_t n, size_t size) 739 739 { 740 740 - return __vmalloc_array(n, size, GFP_KERNEL); 740 740 + return __vmalloc_array_noprof(n, size, GFP_KERNEL); 741 741 } 742 742 EXPORT_SYMBOL(vmalloc_array_noprof); 743 743 ··· 749 749 */ 750 750 void *__vcalloc_noprof(size_t n, size_t size, gfp_t flags) 751 751 { 752 752 - return __vmalloc_array(n, size, flags | __GFP_ZERO); 752 752 + return __vmalloc_array_noprof(n, size, flags | __GFP_ZERO); 753 753 } 754 754 EXPORT_SYMBOL(__vcalloc_noprof); 755 755 ··· 760 760 */ 761 761 void *vcalloc_noprof(size_t n, size_t size) 762 762 { 763 763 - return __vmalloc_array(n, size, GFP_KERNEL | __GFP_ZERO); 763 763 + return __vmalloc_array_noprof(n, size, GFP_KERNEL | __GFP_ZERO); 764 764 } 765 765 EXPORT_SYMBOL(vcalloc_noprof); 766 766

+1 -1

mm/vmalloc.c

reviewed

··· 722 722 * and fall back on vmalloc() if that fails. Others 723 723 * just put it in the vmalloc space. 724 724 */ 725 725 - #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 725 725 + #if defined(CONFIG_EXECMEM) && defined(MODULES_VADDR) 726 726 unsigned long addr = (unsigned long)kasan_reset_tag(x); 727 727 if (addr >= MODULES_VADDR && addr < MODULES_END) 728 728 return 1;

+1 -1

mm/vmscan.c

reviewed

··· 1227 1227 THP_SWPOUT_FALLBACK, 1); 1228 1228 count_vm_event(THP_SWPOUT_FALLBACK); 1229 1229 } 1230 1230 - count_mthp_stat(order, MTHP_STAT_ANON_SWPOUT_FALLBACK); 1230 1230 + count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK); 1231 1231 #endif 1232 1232 if (!add_to_swap(folio)) 1233 1233 goto activate_locked_split;