tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm: cache struct page for empty_zero_page and return it from ZERO_PAGE()

For most architectures every invocation of ZERO_PAGE() does
virt_to_page(empty_zero_page). But empty_zero_page is in BSS and it is
enough to get its struct page once at initialization time and then use it
whenever a zero page should be accessed.

Add yet another __zero_page variable that will be initialized as
virt_to_page(empty_zero_page) for most architectures in a weak
arch_setup_zero_pages() function.

For architectures that use colored zero pages (MIPS and s390) rename their
setup_zero_pages() to arch_setup_zero_pages() and make it global rather
than static.

For architectures that cannot use virt_to_page() for BSS (arm64 and
sparc64) add override of arch_setup_zero_pages().

Link: https://lkml.kernel.org/r/20260211103141.3215197-5-rppt@kernel.org
Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Acked-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Cc: Andreas Larsson <andreas@gaisler.com>
Cc: "Borislav Petkov (AMD)" <bp@alien8.de>
Cc: Christophe Leroy (CS GROUP) <chleroy@kernel.org>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Guo Ren <guoren@kernel.org>
Cc: Helge Deller <deller@gmx.de>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: John Paul Adrian Glaubitz <glaubitz@physik.fu-berlin.de>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Magnus Lindholm <linmag7@gmail.com>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Max Filippov <jcmvbkbc@gmail.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Richard Weinberger <richard@nod.at>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stafford Horne <shorne@gmail.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vineet Gupta <vgupta@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Mike Rapoport (Microsoft) and committed by
Andrew Morton 26513781 6215d9f4

+40 -40
-6
arch/arm64/include/asm/pgtable.h
··· 106 106 #define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ 107 107 local_flush_tlb_page_nonotify(vma, address) 108 108 109 - /* 110 - * ZERO_PAGE is a global shared page that is always zero: used 111 - * for zero-mapped memory areas etc.. 112 - */ 113 - #define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page)) 114 - 115 109 #define pte_ERROR(e) \ 116 110 pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e)) 117 111
+5
arch/arm64/mm/init.c
··· 328 328 memblock_dump_all(); 329 329 } 330 330 331 + void __init arch_setup_zero_pages(void) 332 + { 333 + __zero_page = phys_to_page(__pa_symbol(empty_zero_page)); 334 + } 335 + 331 336 void __init arch_mm_preinit(void) 332 337 { 333 338 unsigned int flags = SWIOTLB_VERBOSE;
+1 -10
arch/mips/mm/init.c
··· 56 56 EXPORT_SYMBOL_GPL(empty_zero_page); 57 57 EXPORT_SYMBOL(zero_page_mask); 58 58 59 - /* 60 - * Not static inline because used by IP27 special magic initialization code 61 - */ 62 - static void __init setup_zero_pages(void) 59 + void __init arch_setup_zero_pages(void) 63 60 { 64 61 unsigned int order; 65 62 ··· 447 450 BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (PFN_PTE_SHIFT > PAGE_SHIFT)); 448 451 449 452 maar_init(); 450 - setup_zero_pages(); /* Setup zeroed pages. */ 451 453 highmem_init(); 452 454 453 455 #ifdef CONFIG_64BIT ··· 456 460 kclist_add(&kcore_kseg0, (void *) CKSEG0, 457 461 0x80000000 - 4, KCORE_TEXT); 458 462 #endif 459 - } 460 - #else /* CONFIG_NUMA */ 461 - void __init arch_mm_preinit(void) 462 - { 463 - setup_zero_pages(); /* This comes from node 0 */ 464 463 } 465 464 #endif /* !CONFIG_NUMA */ 466 465
+1 -3
arch/s390/mm/init.c
··· 69 69 EXPORT_SYMBOL(empty_zero_page); 70 70 EXPORT_SYMBOL(zero_page_mask); 71 71 72 - static void __init setup_zero_pages(void) 72 + void __init arch_setup_zero_pages(void) 73 73 { 74 74 unsigned long total_pages = memblock_estimated_nr_free_pages(); 75 75 unsigned int order; ··· 159 159 cpumask_set_cpu(0, mm_cpumask(&init_mm)); 160 160 161 161 pv_init(); 162 - 163 - setup_zero_pages(); /* Setup zeroed pages. */ 164 162 } 165 163 166 164 unsigned long memory_block_size_bytes(void)
-3
arch/sparc/include/asm/pgtable_64.h
··· 210 210 extern unsigned long pg_iobits; 211 211 extern unsigned long _PAGE_ALL_SZ_BITS; 212 212 213 - extern struct page *mem_map_zero; 214 - #define ZERO_PAGE(vaddr) (mem_map_zero) 215 - 216 213 /* PFNs are real physical page numbers. However, mem_map only begins to record 217 214 * per-page information starting at pfn_base. This is to handle systems where 218 215 * the first physical page in the machine is at some huge physical address,
+7 -10
arch/sparc/mm/init_64.c
··· 177 177 extern unsigned int sparc_ramdisk_image; 178 178 extern unsigned int sparc_ramdisk_size; 179 179 180 - struct page *mem_map_zero __read_mostly; 181 - EXPORT_SYMBOL(mem_map_zero); 182 - 183 180 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly; 184 181 185 182 unsigned long sparc64_kern_pri_context __read_mostly; ··· 2487 2490 register_page_bootmem_info_node(NODE_DATA(i)); 2488 2491 #endif 2489 2492 } 2490 - void __init mem_init(void) 2493 + 2494 + void __init arch_setup_zero_pages(void) 2491 2495 { 2492 2496 phys_addr_t zero_page_pa = kern_base + 2493 2497 ((unsigned long)&empty_zero_page[0] - KERNBASE); 2494 2498 2499 + __zero_page = phys_to_page(zero_page_pa); 2500 + } 2501 + 2502 + void __init mem_init(void) 2503 + { 2495 2504 /* 2496 2505 * Must be done after boot memory is put on freelist, because here we 2497 2506 * might set fields in deferred struct pages that have not yet been ··· 2505 2502 * deferred pages for us. 2506 2503 */ 2507 2504 register_page_bootmem_info(); 2508 - 2509 - /* 2510 - * Set up the zero page, mark it reserved, so that page count 2511 - * is not manipulated when freeing the page from user ptes. 2512 - */ 2513 - mem_map_zero = pfn_to_page(PHYS_PFN(zero_page_pa)); 2514 2505 2515 2506 if (tlb_type == cheetah || tlb_type == cheetah_plus) 2516 2507 cheetah_ecache_flush_init();
+8 -3
include/linux/pgtable.h
··· 1929 1929 * For architectures that don't __HAVE_COLOR_ZERO_PAGE the zero page lives in 1930 1930 * empty_zero_page in BSS. 1931 1931 */ 1932 + void arch_setup_zero_pages(void); 1933 + 1932 1934 #ifdef __HAVE_COLOR_ZERO_PAGE 1933 1935 static inline int is_zero_pfn(unsigned long pfn) 1934 1936 { ··· 1958 1956 } 1959 1957 1960 1958 extern uint8_t empty_zero_page[PAGE_SIZE]; 1959 + extern struct page *__zero_page; 1961 1960 1962 - #ifndef ZERO_PAGE 1963 - #define ZERO_PAGE(vaddr) ((void)(vaddr),virt_to_page(empty_zero_page)) 1964 - #endif 1961 + static inline struct page *_zero_page(unsigned long addr) 1962 + { 1963 + return __zero_page; 1964 + } 1965 + #define ZERO_PAGE(vaddr) _zero_page(vaddr) 1965 1966 1966 1967 #endif /* __HAVE_COLOR_ZERO_PAGE */ 1967 1968
+18 -5
mm/mm_init.c
··· 59 59 #ifndef __HAVE_COLOR_ZERO_PAGE 60 60 uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; 61 61 EXPORT_SYMBOL(empty_zero_page); 62 - #endif 62 + 63 + struct page *__zero_page __ro_after_init; 64 + EXPORT_SYMBOL(__zero_page); 65 + #endif /* __HAVE_COLOR_ZERO_PAGE */ 63 66 64 67 #ifdef CONFIG_DEBUG_MEMORY_INIT 65 68 int __meminitdata mminit_loglevel; ··· 2683 2680 ); 2684 2681 } 2685 2682 2686 - static int __init init_zero_page_pfn(void) 2683 + #ifndef __HAVE_COLOR_ZERO_PAGE 2684 + /* 2685 + * architectures that __HAVE_COLOR_ZERO_PAGE must define this function 2686 + */ 2687 + void __init __weak arch_setup_zero_pages(void) 2687 2688 { 2688 - zero_page_pfn = page_to_pfn(ZERO_PAGE(0)); 2689 - return 0; 2689 + __zero_page = virt_to_page(empty_zero_page); 2690 2690 } 2691 - early_initcall(init_zero_page_pfn); 2691 + #endif 2692 + 2693 + static void __init init_zero_page_pfn(void) 2694 + { 2695 + arch_setup_zero_pages(); 2696 + zero_page_pfn = page_to_pfn(ZERO_PAGE(0)); 2697 + } 2692 2698 2693 2699 void __init __weak arch_mm_preinit(void) 2694 2700 { ··· 2721 2709 void __init mm_core_init(void) 2722 2710 { 2723 2711 arch_mm_preinit(); 2712 + init_zero_page_pfn(); 2724 2713 2725 2714 /* Initializations relying on SMP setup */ 2726 2715 BUILD_BUG_ON(MAX_ZONELISTS > 2);